web search done right
download sample warc file if not present on local run
Mikkel Denker082957f73235
diff --git a/justfile b/justfileindex b22e9d7e..7a04ac2d 100644--- a/justfile+++ b/justfile@@ -12,6 +12,7 @@cargo watch -x 'run -- frontend data/index queries_us.csv'@local:+ wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gzrm -rf data/index/cargo run --release -- indexer local configs/indexer/local.tomlmv data/index/CC-MAIN-20220116093137-20220116123137-00049.warc.gz/* data/index/