An open source, non-profit web search engine
Disable LTR for now to save CPU
Daoud Clarke232a67ddc21a
diff --git a/analyse/load_test.py b/analyse/load_test.pyindex e69de29..82d3bcb 100644--- a/analyse/load_test.py+++ b/analyse/load_test.py@@ -0,0 +1,41 @@+from datetime import datetime, timezone+from multiprocessing.pool import ThreadPool+from pathlib import Path++import numpy as np+import pandas as pd+import requests++NUM_THREADS = 5++QUERIES_PATH = Path(__file__).parent.parent / 'devdata' / 'rankeval-2024-06' / 'queries.csv'+QUERY_URL = "http://localhost:5000/?q={query}"+++np.random.seed(3)+++def run():+ queries = pd.read_csv(QUERIES_PATH)+ query_sample = queries.sample(500)["suggestion"]+ print(query_sample)++ with ThreadPool(NUM_THREADS) as pool:+ times = pool.map(run_query, query_sample)++ print("Num threads\tNum queries\tMean seconds\tStd seconds\tMin seconds\tMax seconds")+ print(f"{NUM_THREADS}\t{len(query_sample)}\t{np.mean(times):.4f}\t{np.std(times):.4f}\t{np.min(times):.4f}\t{np.max(times):.4f}")+++def run_query(query):+ start_time = datetime.now(timezone.utc)+ url = QUERY_URL.format(query=query)+ response = requests.get(url)+ print("Response", response.text[:100])+ end_time = datetime.now(timezone.utc)+ return (end_time - start_time).total_seconds()+++if __name__ == '__main__':+ run()+diff --git a/mwmbl/main.py b/mwmbl/main.pyindex f6e4415..1285d23 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -33,7 +33,7 @@ def run():elif mwmbl_app == "count_urls":count_urls_continuously()elif mwmbl_app == "server":- uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000)+ uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000, workers=10)else:raise ValueError(f"Unknown MWMBL_APP: {mwmbl_app}")diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex ddb0a68..4b749f0 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -12,7 +12,7 @@ from mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.ltr_rank import LTRRanker-+from mwmbl.tinysearchengine.rank import HeuristicAndWikiRankerCURATED_DOMAINS_CACHE_KEY = "curated-domains"CURATED_DOMAINS_CACHE_TIMEOUT = 300@@ -35,6 +35,7 @@ tiny_index.__enter__()model_path = Path(__file__).parent / "resources" / "model.pickle"model = pickle.load(open(model_path, 'rb'))-ranker = LTRRanker(tiny_index, completer, model, 1000, True, 5)+# ranker = LTRRanker(tiny_index, completer, model, 1000, True, 5)+ranker = HeuristicAndWikiRanker(tiny_index, completer)batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 144e16c..baf75fe 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -129,8 +129,8 @@ def get_features(terms, title, url, extract, score, is_complete):features[f'match_terms_{name}'] = match_termsfeatures[f'match_term_proportion_{name}'] = match_terms / len(terms)- tf_idf_features = get_tf_idf_features(match_counts)- features.update({f"{name}_{k}": v for k, v in tf_idf_features.items()})+ # tf_idf_features = get_tf_idf_features(match_counts)+ # features.update({f"{name}_{k}": v for k, v in tf_idf_features.items()})features['num_terms'] = len(terms)features['num_chars'] = len(' '.join(terms))