Skip to content

web search done right

actually load highlightjs when needed

Mikkel Denkerbdd6bc067405
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index e6be88c0..2d35f966 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,13 +1,13 @@
centrality_store_path = "./data/centrality"
-limit_warc_files = 1
+limit_warc_files = 10
download_images = false
minimum_clean_words = 40
# crawl_stability_path = "data/crawl-stability"
[warc_source]
-type = "Local"
-folder = "./data"
-names = ["sample.warc.gz"]
-# type = "HTTP"
-# base_url = "https://data.commoncrawl.org/"
-# warc_paths_file = "data/warc.paths"
+# type = "Local"
+# folder = "./data"
+# names = ["sample.warc.gz"]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "data/warc.paths"
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index bbdc0389..9d3a11c2 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -1,4 +1,4 @@
-limit_warc_files = 5
+limit_warc_files = 10
graph_base_path = "data/webgraph"
batch_size = 1
diff --git a/frontend/src/askama.ts b/frontend/src/askama.ts
index ed0023ba..a0d01b33 100644
--- a/frontend/src/askama.ts
+++ b/frontend/src/askama.ts
@@ -47,11 +47,11 @@ const if_ = (pat: string, then: Component, else_: null | Component = null) => {
export const askama = Object.assign(
(input: TemplateStringsArray) => {
const str = input.raw.join("");
- const segmetns = str.split("$");
- if (passthrough && segmetns.length > 1) {
- return faker.helpers.fake(segmetns.slice(1).join("$"));
+ const segments = str.split("$");
+ if (passthrough && segments.length > 1) {
+ return faker.helpers.fake(segments.slice(1).join("$"));
} else {
- return `{{ ${segmetns[0]} }}`;
+ return `{{ ${segments[0]} }}`;
}
},
{ fake, match, for_, if_ }
diff --git a/frontend/src/layouts/Layout.astro b/frontend/src/layouts/Layout.astro
index 6ae84441..0486b6a9 100644
--- a/frontend/src/layouts/Layout.astro
+++ b/frontend/src/layouts/Layout.astro
@@ -20,10 +20,6 @@ const { title } = Astro.props;
href="/opensearch.xml"
/>
<title>{title}</title>
- </head>
- <body class="font-light">
- <slot />
-
<style>
html,
body {
@@ -82,5 +78,9 @@ const { title } = Astro.props;
appearance: none;
}
</style>
+ </head>
+ <body class="font-light">
+ <slot />
+
</body>
</html>
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 1e444d82..1706959b 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -222,13 +222,15 @@ import { askama } from "../askama";
</noscript>
{
- askama.if_("has_code", () => {
- <link rel="stylesheet" href="/css/highlightjs-default.min.css" />
- <script is:inline src="/js/highlight.min.js"></script>
- <script is:inline>
- hljs.highlightAll();
- </script>
- })
+ askama.if_("has_code", () => (
+ <>
+ <link rel="stylesheet" href="/css/highlightjs-default.min.css" />
+ <script is:inline src="/js/highlight.min.js"></script>
+ <script is:inline>
+ hljs.highlightAll();
+ </script>
+ </>
+ ))
}
<script is:inline>
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index 593ba072..4b63f7ea 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -29,13 +29,13 @@ query: q learning algorithm
result: {domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange", snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}
-score: 1.0
+score: 0.9
query: learn python
result: {domain: "jqxxj.fulltime-abbigliamento.it", title: "Vim plugins python", snippet: "In fact, VimL can be learned fast, but using. omplete Python syntax highlighter for Vim. shows what I get with default Vim that illustrates … Upgrade to PyCharm, the leading Python IDE: best in class debugging, code navigation, refactoring."}
-score: 0.1
+score: 0.0
query: savings interest formula