Skip to content

web search done right

Stackoverflow snippet box text cutoff

Mikkel Denker20f822dde405
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index a789af50..e6be88c0 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -10,4 +10,4 @@ folder = "./data"
names = ["sample.warc.gz"]
# type = "HTTP"
# base_url = "https://data.commoncrawl.org/"
-# warc_paths_file = "warc.paths"
+# warc_paths_file = "data/warc.paths"
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 3ad6087a..7543c6fd 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -34,8 +34,8 @@ use whatlang::Lang;
/// This might require us to store each paragraph of the webpage separately to get adequate performance (maybe we can split passages online with adequate performance
/// but we need to test this).
-const DESIRED_NUM_CHARS: usize = 225;
-const DELTA_NUM_CHARS: usize = 25;
+const DESIRED_NUM_CHARS: usize = 275;
+const DELTA_NUM_CHARS: usize = 50;
const MIN_PASSAGE_WIDTH: usize = 20;
const K1: f64 = 1.2;
@@ -131,17 +131,11 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
.collect();
if passages.is_empty() {
- let above_upper = text.len() > DESIRED_NUM_CHARS + DELTA_NUM_CHARS;
-
let mut snippet = SnippetString {
fragment: text.chars().take(DESIRED_NUM_CHARS).collect(),
highlighted: Vec::new(),
};
- if !snippet.fragment.is_empty() && above_upper {
- snippet.fragment += "...";
- }
-
snippet.highlight(&terms, lang);
return snippet;
@@ -205,7 +199,6 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
.chars()
.take(DESIRED_NUM_CHARS + DELTA_NUM_CHARS)
.collect();
- snippet.fragment += "...";
} else {
let mut next_passage_idx = best_idx + 1;
@@ -223,7 +216,6 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
.chars()
.take(DESIRED_NUM_CHARS + DELTA_NUM_CHARS)
.collect();
- snippet.fragment += "...";
}
}
snippet.highlight(&terms, lang);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 7e3b3fc6..ac5b04da 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -760,7 +760,6 @@ mod test {
let graph = test_graph();
let path = graph.path.clone();
- dbg!(&path);
let frozen: FrozenWebgraph = graph.into();
let bytes = bincode::serialize(&frozen).unwrap();
@@ -768,7 +767,6 @@ mod test {
let deserialized_frozen: FrozenWebgraph = bincode::deserialize(&bytes).unwrap();
let graph: Webgraph = deserialized_frozen.into();
- dbg!(&graph.path);
let distances = graph.distances(Node::from("D"));
diff --git a/frontend/package.json b/frontend/package.json
index e17bf031..0588aa2e 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -23,8 +23,9 @@
"@faker-js/faker": "^7.6.0",
"@fontsource/montserrat": "^4.5.14",
"@tailwindcss/typography": "^0.5.9",
+ "@tailwindcss/line-clamp": "^0.4.2",
"prettier": "^2.8.3",
"prettier-plugin-astro": "^0.8.0",
"prettier-plugin-tailwindcss": "^0.2.2"
}
-}
+}
\ No newline at end of file
diff --git a/frontend/src/components/Snippet.astro b/frontend/src/components/Snippet.astro
index b99188ed..cc5d72d4 100644
--- a/frontend/src/components/Snippet.astro
+++ b/frontend/src/components/Snippet.astro
@@ -12,7 +12,7 @@ export interface Props {
{
askama.match("item.snippet", {
"Snippet::Normal { date, text }": (
- <div>
+ <div class="line-clamp-3">
{askama.if_("let Some(date) = date", () => (
<span class="text-gray-500">{askama`date $ 2. May 2022`}</span>
))}
@@ -24,7 +24,7 @@ export interface Props {
"Snippet::StackOverflowQA { question, answers }": (
<div>
<div>
- <div class="max-h-16 overflow-hidden break-words">
+ <div class="max-h-16 line-clamp-3 overflow-hidden break-words">
{askama.for_("part in question.body", () =>
askama.match("part", {
"CodeOrText::Text(text)": <span>{askama`text`}</span>,
@@ -33,7 +33,7 @@ export interface Props {
)}
</div>
</div>
- <div class="flex space-x-2 text-sm">
+ <div class="flex space-x-2 text-sm mt-2">
{askama.for_("answer in answers", () => (
<a
class="h-52 w-1/3 rounded-lg bg-slate-100 p-5 hover:cursor-pointer hover:bg-slate-600 hover:text-white hover:no-underline"
@@ -56,7 +56,7 @@ export interface Props {
))}
</div>
</div>
- <div>
+ <div class="line-clamp-6">
{askama.for_("part in answer.body", () => (
<div class="select-none">
{askama.match("part", {
diff --git a/frontend/src/components/StackOverflow.astro b/frontend/src/components/StackOverflow.astro
index db8e34af..d601719d 100644
--- a/frontend/src/components/StackOverflow.astro
+++ b/frontend/src/components/StackOverflow.astro
@@ -8,7 +8,7 @@ export interface Props {
}
---
-<div class="flex max-w-lg flex-col space-y-5 rounded-lg border p-5">
+<div class="flex max-w-lg flex-col space-y-5 rounded-lg border p-5 overflow-hidden">
<div class="flex flex-col space-y-1">
<div class="flex grow justify-between space-x-2">
<div>
@@ -38,7 +38,7 @@ export interface Props {
{askama.match("part", {
"CodeOrText::Text(text)": <span>{askama`text`}</span>,
"CodeOrText::Code(code)": (
- <pre class="rounded-lg bg-slate-50"><code style="background: none">{askama`code`}</code></pre>
+ <pre class="rounded-lg bg-slate-50 overflow-scroll"><code style="background: none">{askama`code`}</code></pre>
),
})}
</div>
diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs
index 3e331733..be01ab99 100644
--- a/frontend/tailwind.config.cjs
+++ b/frontend/tailwind.config.cjs
@@ -21,5 +21,5 @@ module.exports = {
},
},
},
- plugins: [require("@tailwindcss/typography")],
+ plugins: [require("@tailwindcss/typography"), require("@tailwindcss/line-clamp")],
};