An open source, non-profit web search engine
Post results
Daoud Clarkeebfc0ecaf3d1
diff --git a/crawl.py b/crawl.pyindex 043b447..c1f6890 100644--- a/crawl.py+++ b/crawl.py@@ -5,6 +5,7 @@ from multiprocessing import Processfrom pathlib import Pathimport django+import requestsfrom django.conf import settingsfrom redis import Redis@@ -23,7 +24,7 @@ django.setup()from mwmbl.indexer.update_urls import record_urls_in_databasefrom mwmbl.crawler.retrieve import crawl_batch-from mwmbl.crawler.batch import HashedBatch+from mwmbl.crawler.batch import HashedBatch, Result, Resultsfrom mwmbl.indexer.index_batches import index_batches@@ -39,10 +40,10 @@ url_queue = RedisURLQueue(redis, lambda: set())def run():- for i in range(10):- process = Process(target=process_batch_continuously)- process.start()- time.sleep(5)+ # for i in range(10):+ # process = Process(target=process_batch_continuously)+ # process.start()+ # time.sleep(5)index_process = Process(target=run_indexing_continuously)index_process.start()@@ -66,7 +67,6 @@ def process_batch():js_timestamp = int(time.time() * 1000)batch = HashedBatch.parse_obj({"user_id_hash": user_id, "timestamp": js_timestamp, "items": results})record_urls_in_database([batch], url_queue)- redis = Redis(host='localhost', port=6379, decode_responses=True)# Push the batch into the Redis queuebatch_json = batch.json()@@ -83,9 +83,8 @@ def run_indexing_continuously():def run_indexing():- redis = Redis(host='localhost', port=6379, decode_responses=True)index_path = data_path / settings.INDEX_NAME- batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 100)+ batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 1)if batch_jsons is None:logger.info("No more batches to index. Sleeping for 10 seconds.")time.sleep(10)@@ -106,7 +105,13 @@ def run_indexing():for item in new_items:logger.info(f"New item: {item}")+ result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,+ score=doc.score, term=doc.term, state=doc.state) for doc in new_items]+ results = Results(api_key="test", results=result_items)+ response = requests.post("http://localhost:8000/api/v1/crawler/results", json=results.dict())+ response.raise_for_status()+ print("Response", response.text)diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearchindex 490c48e..4a1f152 100644Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differdiff --git a/mwmbl/admin.py b/mwmbl/admin.pyindex d5f0cc5..a015962 100644--- a/mwmbl/admin.py+++ b/mwmbl/admin.py@@ -2,10 +2,11 @@ from django.contrib import adminfrom django.contrib.admin import ModelAdminfrom django.contrib.auth.admin import UserAdmin-from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission+from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKeyadmin.site.register(MwmblUser, UserAdmin)admin.site.register(Curation, ModelAdmin)admin.site.register(OldIndex, ModelAdmin)admin.site.register(FlagCuration, ModelAdmin)admin.site.register(DomainSubmission, ModelAdmin)+admin.site.register(ApiKey, ModelAdmin)diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex c564722..cd261e6 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -9,15 +9,18 @@ from uuid import uuid4import boto3import requests+from django.conf import settingsfrom fastapi import HTTPException-from ninja import NinjaAPI+from ninja import NinjaAPI, Schemafrom redis import Redis-from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch+from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Resultsfrom mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.database import Databasefrom mwmbl.indexer.batch_cache import BatchCache+from mwmbl.indexer.index_batches import index_documentsfrom mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus+from mwmbl.models import ApiKeyfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.settings import (ENDPOINT_URL,@@ -31,6 +34,7 @@ from mwmbl.settings import (PUBLIC_USER_ID_LENGTH,FILE_NAME_SUFFIX,DATE_REGEX)+from mwmbl.tinysearchengine.indexer import Document, TinyIndexstats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))@@ -79,25 +83,14 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiof" {invalid_urls}. To suggest a domain to crawl, please visit "f"https://mwmbl.org/app/domain-submissions/new")- now = datetime.now(timezone.utc)- seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds-- # How to pad a string with zeros: https://stackoverflow.com/a/39402910- # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough- padded_seconds = str(seconds).zfill(5)-- # See discussion here: https://stackoverflow.com/a/13484764- uid = str(uuid4())[:8]- filename = f'1/{VERSION}/{now.date()}/1/{user_id_hash}/{padded_seconds}__{uid}.json.gz'-# Using an approach from https://stackoverflow.com/a/30476450+ now = datetime.now(timezone.utc)epoch_time = (now - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()hashed_batch = HashedBatch(user_id_hash=user_id_hash, timestamp=epoch_time, items=batch.items)stats_manager.record_batch(hashed_batch)- data = gzip.compress(hashed_batch.json().encode('utf8'))- upload(data, filename)+ filename = upload_object(hashed_batch, now, user_id_hash, "batch")global last_batchlast_batch = hashed_batch@@ -118,6 +111,21 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'url': batch_url,}+ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):+ seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds++ # How to pad a string with zeros: https://stackoverflow.com/a/39402910+ # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough+ padded_seconds = str(seconds).zfill(5)++ # See discussion here: https://stackoverflow.com/a/13484764+ uid = str(uuid4())[:8]++ filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'+ data = gzip.compress(model_object.json().encode('utf8'))+ upload(data, filename)+ return filename+@router.post('/batches/new')def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:user_id_hash = _get_user_id_hash(batch_request)@@ -163,6 +171,25 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'status': 'ok'}+ @router.post('/results')+ def post_results(request, results: Results):+ # Check the API key+ api_key = ApiKey.objects.get(key=results.api_key)+ if api_key is None:+ raise HTTPException(401, "Invalid API key")++ documents = [Document(url=result.url, title=result.title, extract=result.extract) for result in results.results]+ index_path = f"{settings.DATA_PATH}/{settings.INDEX_NAME}"+ index_documents(documents, index_path)++ now = datetime.now(timezone.utc)+ filename = upload_object(results, now, api_key.user.username, "results")++ return {+ 'status': 'ok',+ 'url': f'{PUBLIC_URL_PREFIX}{filename}',+ }+return routerdiff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex 2380e49..433fa2e 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -58,3 +58,17 @@ class HashedBatch(Schema):user_id_hash: strtimestamp: intitems: list[Item]+++class Result(Schema):+ url: str+ title: str+ extract: str+ score: Optional[float]+ term: Optional[str]+ state: Optional[int]+++class Results(Schema):+ api_key: str+ results: list[Result]diff --git a/mwmbl/database.py b/mwmbl/database.pyindex c55cd11..10d874b 100644--- a/mwmbl/database.py+++ b/mwmbl/database.py@@ -1,6 +1,7 @@-from django.conf import settingsfrom psycopg2 import connect+from mwmbl import settings+class Database:def __init__(self):diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex c221ade..480856a 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -46,11 +46,16 @@ def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Countstart_time = datetime.utcnow()document_tuples = list(get_documents_from_batches(batch_data))documents = [Document(title, url, extract) for title, url, extract in document_tuples]+ end_time, new_page_doc_counts = index_documents(documents, index_path)+ logger.info(f"Indexing took {end_time - start_time}")+ return new_page_doc_counts+++def index_documents(documents, index_path):page_documents = preprocess_documents(documents, index_path)new_page_doc_counts = index_pages(index_path, page_documents)end_time = datetime.utcnow()- logger.info(f"Indexing took {end_time - start_time}")- return new_page_doc_counts+ return end_time, new_page_doc_countsdef index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> Counter:diff --git a/mwmbl/migrations/0010_apikey.py b/mwmbl/migrations/0010_apikey.pynew file mode 100644index 0000000..505bb5d--- /dev/null+++ b/mwmbl/migrations/0010_apikey.py@@ -0,0 +1,25 @@+# Generated by Django 4.2.14 on 2024-08-09 18:21++from django.conf import settings+from django.db import migrations, models+import django.db.models.deletion+import mwmbl.models+++class Migration(migrations.Migration):++ dependencies = [+ ('mwmbl', '0009_alter_domainsubmission_options'),+ ]++ operations = [+ migrations.CreateModel(+ name='ApiKey',+ fields=[+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),+ ('key', models.CharField(default=mwmbl.models.random_api_key, max_length=300, unique=True)),+ ('created_on', models.DateTimeField()),+ ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ],+ ),+ ]diff --git a/mwmbl/models.py b/mwmbl/models.pyindex e996791..91026dc 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -1,3 +1,5 @@+import secrets+from django.db import modelsfrom django.contrib.auth.models import AbstractUser@@ -93,3 +95,13 @@ class DomainSubmission(models.Model):status_changed_on = models.DateTimeField(null=True, blank=True)rejection_reason = models.CharField(max_length=20, choices=[(k, v) for k, v in DOMAIN_REJECTION_REASON.items()], blank=True)rejection_detail = models.CharField(max_length=300, blank=True)+++def random_api_key():+ return secrets.token_urlsafe(64)+++class ApiKey(models.Model):+ user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)+ key = models.CharField(max_length=300, unique=True, default=random_api_key)+ created_on = models.DateTimeField()diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 2540eec..b3e4605 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -20,9 +20,11 @@ from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentStatefrom mwmbl.tokenizer import tokenize, get_bigramsfrom mwmbl.utils import request_cache+logger = getLogger(__name__)+MAX_QUERY_CHARS = 100MATCH_SCORE_THRESHOLD = 0.0SCORE_THRESHOLD = 0.0LENGTH_PENALTY = 0.04@@ -340,10 +342,17 @@ def clean_html(s: str):return html.unescape(HTML_TAG_REGEX.sub('', s))-def get_wiki_results(s: str, max_wiki_results: int):+def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:escaped_query = urllib.parse.quote(s, safe='')with request_cache(timedelta(weeks=10)) as session:wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()++ if 'query' not in wiki_response or 'search' not in wiki_response['query']:+ if 'error' in wiki_response:+ logger.warning("Error in wiki response: %s", wiki_response['error'])++ return []+wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)for i, result in enumerate(wiki_response['query']['search'][:max_wiki_results])]@@ -364,12 +373,14 @@ class HeuristicAndWikiRanker(HeuristicRanker):self.max_wiki_results = max_wiki_resultsdef search(self, s: str, additional_results: list[Document]) -> list[Document]:- results = super().search(s, additional_results)+ s_shortened = s[:MAX_QUERY_CHARS]++ results = super().search(s_shortened, additional_results)if len(results) == 0 and self.return_none_if_no_mwmbl_results:return []- wiki_results = get_wiki_results(s, self.max_wiki_results)+ wiki_results = get_wiki_results(s_shortened, self.max_wiki_results)return wiki_results + resultsdiff --git a/mwmbl/views.py b/mwmbl/views.pyindex 5107ed3..88ff881 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -64,10 +64,6 @@ def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,return paragraphs, title-class Optionallist:- pass--def _prepare_results(results: Optional[list[Document]]) -> Optional[dict[str, list[Document]]]:if results is None:return None