Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Disable LTR for now to save CPU

Daoud Clarke232a67ddc21a
diff --git a/analyse/load_test.py b/analyse/load_test.py
index e69de29..82d3bcb 100644
--- a/analyse/load_test.py
+++ b/analyse/load_test.py
@@ -0,0 +1,41 @@
+from datetime import datetime, timezone
+from multiprocessing.pool import ThreadPool
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import requests
+
+NUM_THREADS = 5
+
+QUERIES_PATH = Path(__file__).parent.parent / 'devdata' / 'rankeval-2024-06' / 'queries.csv'
+QUERY_URL = "http://localhost:5000/?q={query}"
+
+
+np.random.seed(3)
+
+
+def run():
+ queries = pd.read_csv(QUERIES_PATH)
+ query_sample = queries.sample(500)["suggestion"]
+ print(query_sample)
+
+ with ThreadPool(NUM_THREADS) as pool:
+ times = pool.map(run_query, query_sample)
+
+ print("Num threads\tNum queries\tMean seconds\tStd seconds\tMin seconds\tMax seconds")
+ print(f"{NUM_THREADS}\t{len(query_sample)}\t{np.mean(times):.4f}\t{np.std(times):.4f}\t{np.min(times):.4f}\t{np.max(times):.4f}")
+
+
+def run_query(query):
+ start_time = datetime.now(timezone.utc)
+ url = QUERY_URL.format(query=query)
+ response = requests.get(url)
+ print("Response", response.text[:100])
+ end_time = datetime.now(timezone.utc)
+ return (end_time - start_time).total_seconds()
+
+
+if __name__ == '__main__':
+ run()
+
diff --git a/mwmbl/main.py b/mwmbl/main.py
index f6e4415..1285d23 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -33,7 +33,7 @@ def run():
elif mwmbl_app == "count_urls":
count_urls_continuously()
elif mwmbl_app == "server":
- uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000)
+ uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000, workers=10)
else:
raise ValueError(f"Unknown MWMBL_APP: {mwmbl_app}")
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index ddb0a68..4b749f0 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -12,7 +12,7 @@ from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
from mwmbl.tinysearchengine.ltr_rank import LTRRanker
-
+from mwmbl.tinysearchengine.rank import HeuristicAndWikiRanker
CURATED_DOMAINS_CACHE_KEY = "curated-domains"
CURATED_DOMAINS_CACHE_TIMEOUT = 300
@@ -35,6 +35,7 @@ tiny_index.__enter__()
model_path = Path(__file__).parent / "resources" / "model.pickle"
model = pickle.load(open(model_path, 'rb'))
-ranker = LTRRanker(tiny_index, completer, model, 1000, True, 5)
+# ranker = LTRRanker(tiny_index, completer, model, 1000, True, 5)
+ranker = HeuristicAndWikiRanker(tiny_index, completer)
batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 144e16c..baf75fe 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -129,8 +129,8 @@ def get_features(terms, title, url, extract, score, is_complete):
features[f'match_terms_{name}'] = match_terms
features[f'match_term_proportion_{name}'] = match_terms / len(terms)
- tf_idf_features = get_tf_idf_features(match_counts)
- features.update({f"{name}_{k}": v for k, v in tf_idf_features.items()})
+ # tf_idf_features = get_tf_idf_features(match_counts)
+ # features.update({f"{name}_{k}": v for k, v in tf_idf_features.items()})
features['num_terms'] = len(terms)
features['num_chars'] = len(' '.join(terms))