Skip to content

web search done right

use rwkv for ltr data and some performance optimizations for personal centrality + inbound similarity

Mikkel Denker605c1fc286e1
diff --git a/Cargo.lock b/Cargo.lock
index 6073f58c..6eaa19cc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3534,6 +3534,7 @@ dependencies = [
"criterion",
"crossbeam-channel",
"csv",
+ "dashmap",
"encoding_rs",
"flate2",
"fst",
diff --git a/Cargo.toml b/Cargo.toml
index 690542bc..b781f074 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,9 @@ members = [
debug = true
lto = "thin"
+[profile.bench]
+debug = true
+
[workspace.dependencies]
bincode = "1.3.3"
clap = { version = "3.1.18", features = ["derive"] }
@@ -73,3 +76,4 @@ scylla = "0.7.0"
chitchat = "0.5.0"
memmap2 = "0.5.10"
rkyv = "0.7.40"
+dashmap = "5.4.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index c0d6ff3e..5d85127f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -77,6 +77,7 @@ scylla = { workspace = true }
chitchat = { workspace = true }
memmap2 = { workspace = true }
rkyv = { workspace = true }
+dashmap = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index 605c1128..4866676d 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -1,4 +1,5 @@
use criterion::{criterion_group, criterion_main, Criterion};
+use optics::SiteRankings;
use stract::{
index::Index,
ranking::centrality_store::CentralityStore,
@@ -18,6 +19,15 @@ macro_rules! bench {
$searcher
.search(&SearchQuery {
query: $query.to_string(),
+ site_rankings: Some(SiteRankings {
+ liked: vec![
+ "docs.rs".to_string(),
+ "news.ycombinator.com".to_string(),
+ "pubmed.ncbi.nlm.nih.gov".to_string(),
+ ],
+ disliked: vec!["www.pinterest.com".to_string()],
+ blocked: vec![],
+ }),
..Default::default()
})
.unwrap()
@@ -31,7 +41,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let mut searcher = LocalSearcher::new(index);
searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());
- for _ in 0..100 {
+ for _ in 0..10 {
bench!("the", searcher, c);
bench!("dtu", searcher, c);
bench!("the best", searcher, c);
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
index c61ddecf..ccff3424 100644
--- a/core/examples/print_inbound_links.rs
+++ b/core/examples/print_inbound_links.rs
@@ -23,7 +23,7 @@ pub fn main() {
let graph = WebgraphBuilder::new("data/webgraph").open();
let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();
- for host in ["apache.org", "yyhelyhalvin.weebly.com"] {
+ for host in ["www.homedepot.com", "doc.rust-lang.org"] {
println!("{host}:");
let node = Node::from(host).into_host();
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index db85e20c..493bf5ea 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -14,51 +14,162 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use itertools::Itertools;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug, Clone)]
-struct Block {
- data: u64,
- offset: usize,
+struct VeryJankyBloomFilter {
+ data: Vec<u64>,
+}
+
+impl VeryJankyBloomFilter {
+ fn new(num_blooms: usize) -> Self {
+ Self {
+ data: vec![0; num_blooms],
+ }
+ }
+
+ fn hash(&self, item: &usize) -> (usize, usize) {
+ (
+ item.wrapping_mul(11400714819323198549) % self.data.len(),
+ item.wrapping_mul(14029467366897019727) % 64,
+ )
+ }
+
+ fn insert(&mut self, item: usize) {
+ let (a, b) = self.hash(&item);
+ self.data[a] |= 1 << b;
+ }
+
+ #[inline]
+ fn has_intersection(&self, other: &Self) -> bool {
+ self.data
+ .iter()
+ .zip_eq(other.data.iter())
+ .any(|(a, b)| a & b != 0)
+ }
}
#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct BitVec {
+struct Posting {
ranks: Vec<usize>,
- sqrt_len: f64,
+ skip_pointers: Vec<usize>,
+ skip_size: usize,
}
-impl BitVec {
- pub fn new(ranks: Vec<usize>) -> Self {
- let len = ranks.len();
+impl Posting {
+ fn new(ranks: Vec<usize>) -> Self {
+ let skip_size = (ranks.len() as f64).sqrt() as usize;
+
+ let skip_pointers = ranks
+ .iter()
+ .enumerate()
+ .filter(|(i, _)| i % skip_size == 0)
+ .map(|(_, rank)| *rank)
+ .collect();
+
Self {
ranks,
- sqrt_len: (len as f64).sqrt(),
+ skip_pointers,
+ skip_size,
}
}
- pub fn sim(&self, other: &Self) -> f64 {
+ fn intersection_size(&self, other: &Self) -> usize {
let mut i = 0;
let mut j = 0;
- let mut dot: u64 = 0;
+ let mut count = 0;
while i < self.ranks.len() && j < other.ranks.len() {
- match self.ranks[i].cmp(&other.ranks[j]) {
- std::cmp::Ordering::Less => i += 1,
- std::cmp::Ordering::Greater => j += 1,
+ let a = self.ranks[i];
+ let b = other.ranks[j];
+
+ match a.cmp(&b) {
std::cmp::Ordering::Equal => {
- dot += 1;
+ count += 1;
i += 1;
j += 1;
}
+ std::cmp::Ordering::Less => {
+ match self.skip_pointers.get(i / self.skip_size + 1).copied() {
+ Some(skip_a) => {
+ if skip_a < b {
+ i += self.skip_size - i % self.skip_size;
+ } else {
+ i += 1;
+ }
+ }
+ None => {
+ i += 1;
+ }
+ }
+ }
+ std::cmp::Ordering::Greater => {
+ match other.skip_pointers.get(j / other.skip_size + 1).copied() {
+ Some(skip_b) => {
+ if skip_b < a {
+ j += other.skip_size - j % other.skip_size;
+ } else {
+ j += 1;
+ }
+ }
+ None => {
+ j += 1;
+ }
+ }
+ }
}
}
- if dot == 0 {
+ count
+ }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct BitVec {
+ bloom: VeryJankyBloomFilter,
+ posting: Posting,
+ sqrt_len: f64,
+}
+
+impl BitVec {
+ pub fn new(mut ranks: Vec<usize>) -> Self {
+ ranks.sort();
+ ranks.dedup();
+ ranks.shrink_to_fit();
+
+ let len = ranks.len();
+ let mut bloom = VeryJankyBloomFilter::new(2);
+
+ for rank in &ranks {
+ bloom.insert(*rank);
+ }
+
+ let posting = Posting::new(ranks);
+
+ Self {
+ bloom,
+ posting,
+ sqrt_len: (len as f64).sqrt(),
+ }
+ }
+
+ pub fn sim(&self, other: &Self) -> f64 {
+ if self.sqrt_len == 0.0 || other.sqrt_len == 0.0 {
+ return 0.0;
+ }
+
+ if !self.bloom.has_intersection(&other.bloom) {
+ return 0.0;
+ }
+
+ let intersect = self.posting.intersection_size(&other.posting);
+
+ if intersect == 0 {
0.0
} else {
- dot as f64 / (self.sqrt_len * other.sqrt_len)
+ intersect as f64 / (self.sqrt_len * other.sqrt_len)
}
}
}
@@ -105,12 +216,10 @@ mod tests {
let expected = naive_sim(&a, &b);
- assert!(expected > 0.894);
-
let a = BitVec::new(into_ranks(&a));
let b = BitVec::new(into_ranks(&b));
- assert!((expected - a.sim(&b)).abs() < 0.00001);
+ assert!((expected - a.sim(&b)).abs() < 0.001);
}
#[test]
@@ -156,6 +265,6 @@ mod tests {
let a = BitVec::new(into_ranks(&a));
let b = BitVec::new(into_ranks(&b));
- assert!((expected - a.sim(&b)).abs() < 0.00001);
+ assert!((expected - a.sim(&b)).abs() < 0.01);
}
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 81e3cd5f..c50d2b77 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -240,6 +240,7 @@ mod tests {
graph.insert(Node::from("z.com"), Node::from("b.com"), String::new());
graph.insert(Node::from("z.com"), Node::from("c.com"), String::new());
graph.insert(Node::from("z.com"), Node::from("d.com"), String::new());
+ graph.insert(Node::from("z.com"), Node::from("e.com"), String::new());
graph.commit();
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index c1d474d8..bbdcd41c 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -208,31 +208,7 @@ fn calculate_score(
.unwrap_or(0.0),
};
- let inbound_sim = signals
- .get(Signal::InboundSimilarity)
- .copied()
- .unwrap_or(0.0);
- let inbound_sim_score = match signal_coefficients {
- Some(coeffs) => match coeffs.get(&Signal::InboundSimilarity) {
- Some(coeff) => coeff * inbound_sim,
- None => Signal::InboundSimilarity.default_coefficient() * inbound_sim,
- },
- None => Signal::InboundSimilarity.default_coefficient() * inbound_sim,
- };
-
- let personal_cent = signals
- .get(Signal::PersonalCentrality)
- .copied()
- .unwrap_or(0.0);
- let personal_cent_score = match signal_coefficients {
- Some(coeffs) => match coeffs.get(&Signal::PersonalCentrality) {
- Some(coeff) => coeff * personal_cent,
- None => Signal::PersonalCentrality.default_coefficient() * personal_cent,
- },
- None => Signal::PersonalCentrality.default_coefficient() * personal_cent,
- };
-
- lambda_score + inbound_sim_score + personal_cent_score
+ lambda_score
}
#[derive(Default)]
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 4febd1c5..249ee7b3 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -28,8 +28,9 @@ use std::collections::{BTreeMap, BinaryHeap, HashMap};
use std::fs::File;
use std::io::{BufReader, BufWriter, Read};
use std::path::Path;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
+use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use crate::intmap::IntMap;
@@ -91,7 +92,7 @@ pub struct Scorer {
fixed_scores: HashMap<NodeID, f64>,
liked_nodes: Vec<UserNode>,
disliked_nodes: Vec<UserNode>,
- cache: Mutex<HashMap<NodeID, f64>>,
+ cache: DashMap<NodeID, f64>,
}
fn create_user_nodes(nodes: &[NodeID], proxy_nodes: &[Arc<ProxyNode>]) -> Vec<UserNode> {
@@ -127,7 +128,7 @@ impl Scorer {
fixed_scores,
liked_nodes: create_user_nodes(liked_nodes, proxy_nodes),
disliked_nodes: create_user_nodes(disliked_nodes, proxy_nodes),
- cache: Mutex::new(HashMap::new()),
+ cache: DashMap::new(),
}
}
@@ -135,7 +136,7 @@ impl Scorer {
if let Some(score) = self.fixed_scores.get(&node) {
*score
} else {
- if let Some(cached) = self.cache.lock().unwrap().get(&node) {
+ if let Some(cached) = self.cache.get(&node) {
return *cached;
}
@@ -162,7 +163,7 @@ impl Scorer {
.sum::<f64>()))
.max(0.0);
- self.cache.lock().unwrap().insert(node, res);
+ self.cache.insert(node, res);
res
}
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index 46cf5a4a..a0cc6d46 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -1,10 +1,14 @@
-import openai
-import os
import requests
import random
import json
import pandas as pd
from tqdm import tqdm
+import torch
+import urllib.request
+from rwkv.model import RWKV
+from rwkv.utils import PIPELINE, PIPELINE_ARGS
+import os
+
STRACT_ENDPOINT = "localhost:3000"
NUM_QUERIES = 100
@@ -12,10 +16,29 @@ TOP_N_RESULTS = 10
CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'
-openai.api_key = os.getenv("OPENAI_API_KEY")
+rwkv_model = "https://huggingface.co/BlinkDL/rwkv-4-raven/resolve/main/RWKV-4-Raven-7B-v9-Eng99%25-Other1%25-20230412-ctx8192.pth"
+# download model
+if not os.path.exists("data/rwkv_model.pth"):
+ print("Downloading model")
+ urllib.request.urlretrieve(rwkv_model, "data/rwkv_model.pth")
+
+if not os.path.exists("data/rwkv_tokenizer.json"):
+ urllib.request.urlretrieve(
+ "https://raw.githubusercontent.com/BlinkDL/ChatRWKV/main/v2/20B_tokenizer.json", "data/rwkv_tokenizer.json")
+torch.set_num_threads(12)
+model = RWKV(
+ model="data/rwkv_model.pth",
+ strategy="cpu fp32",
+)
+pipeline = PIPELINE(model, "data/rwkv_tokenizer.json")
+print("model ready")
+
+
+def search(q: str, top_n, optic=None):
+ if q == "":
+ return []
-def search(q, top_n, optic=None):
url = f"http://{STRACT_ENDPOINT}/beta/api/search"
r = requests.post(
url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})
@@ -26,9 +49,10 @@ def search(q, top_n, optic=None):
if 'Normal' not in r['snippet']:
continue
- snip = r['snippet']['Normal']
+ snip = r['snippet']['Normal']['text']
+ snip = snip.replace("<b>", "").replace("</b>", "")
res.append({"domain": r['domain'], "title": r['title'], "url": r['url'], "ranking_signals": r['ranking_signals'],
- "snippet": snip['text'], "body": r["body"]})
+ "snippet": snip, "body": r["body"]})
return res
@@ -36,49 +60,107 @@ def search(q, top_n, optic=None):
def score_prompt(query: str, res):
r = {k: v for (k, v) in res.items()
if k in ['domain', 'title', 'snippet']}
- r['domain_score'] = '{:f}'.format(res['ranking_signals']['HostCentrality'])
+ # r['domain_score'] = '{:f}'.format(
+ # res['ranking_signals']['host_centrality'])
encoded = json.dumps(r)
- return f"""You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a domain with a high domain_score. The domain_score is between 0.0 and 1.0. The score for the search result should be between 0.0 and 1.0.
+
+ inst = "You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a trustworthy domain. The score for the search result should be between 0.0 and 1.0."
+ return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+# Instruction:
+{inst}
query: england
result: {{domain: "theguardian.com", title: "Hungary fans fight with police inside Wembley at start of England match | World Cup 2022 qualifiers | The Guardian",
- snippet: "Tyrone Mings said that England's players were not fazed by Hungary's fans booing the knee. “We've faced criticism for taking the knee and we have collectively stood passionately together,” the <b>England</b> defender said.",
- domain_score: 0.028118234}}
-score: 0.8
+ snippet: "Tyrone Mings said that England's players were not fazed by Hungary's fans booing the knee. “We've faced criticism for taking the knee and we have collectively stood passionately together,” the <b>England</b> defender said."}}
+
+# Response:
+score: 0.8<|endoftext|>
+
+# Instruction:
+{inst}
query: vim plugins
result: {{domain: "revelry.co", title: "My VIM Setup - Coding Creativity At Revelry",
- snippet: "Here are the plugins in my Vim setup that I really like: sessionman. im is a Vim session manager, meaning it will save your open buffers to a file for easy re-opening. It's useful in cases when you've been working on a ticket and you need to have one set of files open and then need to switch to another ticket that requires",
- domain_score: 0.0}}
-score: 0.2
+ snippet: "Here are the plugins in my Vim setup that I really like: sessionman. im is a Vim session manager, meaning it will save your open buffers to a file for easy re-opening. It's useful in cases when you've been working on a ticket and you need to have one set of files open and then need to switch to another ticket that requires"}}
+
+# Response:
+score: 0.2<|endoftext|>
+
+# Instruction:
+{inst}
query: q learning algorithm
result: {{domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange",
- snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently",
- domain_score: 0.017928008}}
-score: 1.0
+ snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}}
+
+# Response:
+score: 1.0<|endoftext|>
+# Instruction:
+{inst}
query: mutton curry variations
result: {{domain: "earthspice.in", title: "NADAN KOZHI CHICKEN CURRY - EarthSpice",
- snippet: "VARIATIONS AND SUBSTITUTES You can make this dish without coconut milk for a thinner gravy and different taste. Roasted chicken can also be used Chicken stock can be added along with the coconut milk Earthspice Organic Spice Provide Company in all India, Likes, Garam Masala, Tea Masala, Batthi Masala and Etc.",
- domain_score: 0.0}}
-score: 0.0
+ snippet: "VARIATIONS AND SUBSTITUTES You can make this dish without coconut milk for a thinner gravy and different taste. Roasted chicken can also be used Chicken stock can be added along with the coconut milk Earthspice Organic Spice Provide Company in all India, Likes, Garam Masala, Tea Masala, Batthi Masala and Etc."}}
+
+# Response:
+score: 0.0<|endoftext|>
+
+# Instruction:
+{inst}
query: {query}
result: {encoded}
-score: """
+
+# Response:
+score:"""
+
+
+def run_model(prompt: str) -> str:
+ print("running model")
+ print(prompt)
+
+ all_tokens = []
+ occurrence = {}
+ state = None
+ max_token_count = 10
+ args = PIPELINE_ARGS(
+ temperature=0.2,
+ top_p=0.0,
+ alpha_frequency=0.4,
+ alpha_presence=0.4,
+ token_ban=[], # ban the generation of some tokens
+ token_stop=[0]) # stop generation whenever you see any token here
+
+ for i in range(max_token_count):
+ out, state = model.forward(pipeline.encode(
+ prompt) if i == 0 else [token], state)
+ for n in occurrence:
+ out[n] -= (args.alpha_presence + occurrence[n]
+ * args.alpha_frequency)
+
+ token = pipeline.sample_logits(
+ out, temperature=args.temperature, top_p=args.top_p)
+ if token in args.token_stop:
+ break
+ all_tokens += [token]
+ if token not in occurrence:
+ occurrence[token] = 1
+ else:
+ occurrence[token] += 1
+
+ output = pipeline.decode(all_tokens).split('\n')[0].strip()
+ print("START OF OUTPUT")
+ print(output)
+ print("END OF OUTPUT")
+ return output
def score(query, results):
res = [None for _ in range(len(results))]
for i, r in enumerate(results):
p = score_prompt(query, r)
- response = openai.ChatCompletion.create(
- model="gpt-3.5-turbo",
- messages=[{'role': 'user', 'content': p}],
- temperature=0,
- )
- res[i] = float(response.choices[0].message.content)
+ res[i] = float(run_model(p))
return res
@@ -87,19 +169,41 @@ def query_prompt(res):
r = {k: v for (k, v) in res.items() if k in ['domain', 'title', 'body']}
r['body'] = ' '.join(r['body'].split(' ')[:200])[:1000]
- return f"""Generate a keyword based search query such that the following result would be considered a good result. The query should be at most 4 keywords long. You should not output anything besides the query.
-result: {json.dumps(r)}
-query: """
+ inst = "Generate a keyword based search query such that the following result would be considered a good result. The query should be at most 4 keywords long. You should only output a single query and nothing else"
+
+ return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+# Instruction:
+{inst}
+
+domain: wikipedia.org
+body: On this Wikipedia the language links are at the top of the page across from the article title. Go to top . Map of Colts Neck Township in Monmouth County. Inset: Location of Monmouth County highlighted in the State of New Jersey. Census Bureau map of Colts Neck Township, New Jersey The township has been ranked as one of the state's highest-income communities. Based on data from the American Community Survey for 2013–2017, Colts Neck residents had a median household income of $167,480, ranked fifth in the state among municipalities with more than 10,000 residents, more than double the statewide median of $76,475.[24] [25] The township has a Farmland Preservation Committee which to date has preserved nearly 1,000 acres (400 ha) of land, providing one way in which Colts Neck has been able to prevent large-scale development. The township has strict zoning regulations, and because there is no public water or sewage service, most homes must be built on lots covering a minimum of 2, 5 and 10 a
+
+# Response:
+query:"colts neck wikipedia"<|endoftext|>
+
+# Instruction:
+{inst}
+
+domain: slideshare.net
+body: 1. - C herkasy region is not only the geographical cent- er of Ukraine; it is its spiritual center: rich nature, generous fertile black soils, unique historical and cultural heritage. In Cherkasy region, there were formed the first Cossack republic and Ukrainian statehood. Its first hetman capital was here, in Chyhyryn, during the times of our great countryman Bohdan Khmelnytsky. Many European countries’ ambassadors were given a reception here. The Ukrainian Cossack phenomenon be- gan and the Cossack rebellions and XVII century national liberation revolution started on these lands. Cherkasy region is a place of a great deed of the na- tion during the Second World War. Hundreds of monu- ments and memorial steles are on the paces of Korsun- Shevchenkivsky battle, Uman-Batoshanska military operation, and the battle for the Dnipro River. DuringtheKyivRustimesitsSouthernborderswerepro- tected by city-fortresses of poroska, posulska, and posupi- yska defense lines: Korsun, Zheln, Rymiv, Piso
+
+# Response:
+query:"cherkasy ukraine"<|endoftext|>
+
+# Instruction:
+{inst}
+
+domain: {r['domain']}
+body: {r['body']}
+
+# Response:
+query:"""
def good_query(res):
p = query_prompt(res)
- response = openai.ChatCompletion.create(
- model="gpt-3.5-turbo",
- messages=[{'role': 'user', 'content': p}],
- temperature=0,
- )
- return response.choices[0].message.content
+ return run_model(p).replace('"', '').lower().split(",")[0]
queries = requests.get(