Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Post results

Daoud Clarkeebfc0ecaf3d1
diff --git a/crawl.py b/crawl.py
index 043b447..c1f6890 100644
--- a/crawl.py
+++ b/crawl.py
@@ -5,6 +5,7 @@ from multiprocessing import Process
from pathlib import Path
import django
+import requests
from django.conf import settings
from redis import Redis
@@ -23,7 +24,7 @@ django.setup()
from mwmbl.indexer.update_urls import record_urls_in_database
from mwmbl.crawler.retrieve import crawl_batch
-from mwmbl.crawler.batch import HashedBatch
+from mwmbl.crawler.batch import HashedBatch, Result, Results
from mwmbl.indexer.index_batches import index_batches
@@ -39,10 +40,10 @@ url_queue = RedisURLQueue(redis, lambda: set())
def run():
- for i in range(10):
- process = Process(target=process_batch_continuously)
- process.start()
- time.sleep(5)
+ # for i in range(10):
+ # process = Process(target=process_batch_continuously)
+ # process.start()
+ # time.sleep(5)
index_process = Process(target=run_indexing_continuously)
index_process.start()
@@ -66,7 +67,6 @@ def process_batch():
js_timestamp = int(time.time() * 1000)
batch = HashedBatch.parse_obj({"user_id_hash": user_id, "timestamp": js_timestamp, "items": results})
record_urls_in_database([batch], url_queue)
- redis = Redis(host='localhost', port=6379, decode_responses=True)
# Push the batch into the Redis queue
batch_json = batch.json()
@@ -83,9 +83,8 @@ def run_indexing_continuously():
def run_indexing():
- redis = Redis(host='localhost', port=6379, decode_responses=True)
index_path = data_path / settings.INDEX_NAME
- batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 100)
+ batch_jsons = redis.lpop(BATCH_QUEUE_KEY, 1)
if batch_jsons is None:
logger.info("No more batches to index. Sleeping for 10 seconds.")
time.sleep(10)
@@ -106,7 +105,13 @@ def run_indexing():
for item in new_items:
logger.info(f"New item: {item}")
+ result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,
+ score=doc.score, term=doc.term, state=doc.state) for doc in new_items]
+ results = Results(api_key="test", results=result_items)
+ response = requests.post("http://localhost:8000/api/v1/crawler/results", json=results.dict())
+ response.raise_for_status()
+ print("Response", response.text)
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearch
index 490c48e..4a1f152 100644
Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differ
diff --git a/mwmbl/admin.py b/mwmbl/admin.py
index d5f0cc5..a015962 100644
--- a/mwmbl/admin.py
+++ b/mwmbl/admin.py
@@ -2,10 +2,11 @@ from django.contrib import admin
from django.contrib.admin import ModelAdmin
from django.contrib.auth.admin import UserAdmin
-from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission
+from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKey
admin.site.register(MwmblUser, UserAdmin)
admin.site.register(Curation, ModelAdmin)
admin.site.register(OldIndex, ModelAdmin)
admin.site.register(FlagCuration, ModelAdmin)
admin.site.register(DomainSubmission, ModelAdmin)
+admin.site.register(ApiKey, ModelAdmin)
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index c564722..cd261e6 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -9,15 +9,18 @@ from uuid import uuid4
import boto3
import requests
+from django.conf import settings
from fastapi import HTTPException
-from ninja import NinjaAPI
+from ninja import NinjaAPI, Schema
from redis import Redis
-from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch
+from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results
from mwmbl.crawler.stats import MwmblStats, StatsManager
from mwmbl.database import Database
from mwmbl.indexer.batch_cache import BatchCache
+from mwmbl.indexer.index_batches import index_documents
from mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus
+from mwmbl.models import ApiKey
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.settings import (
ENDPOINT_URL,
@@ -31,6 +34,7 @@ from mwmbl.settings import (
PUBLIC_USER_ID_LENGTH,
FILE_NAME_SUFFIX,
DATE_REGEX)
+from mwmbl.tinysearchengine.indexer import Document, TinyIndex
stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))
@@ -79,25 +83,14 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
f" {invalid_urls}. To suggest a domain to crawl, please visit "
f"https://mwmbl.org/app/domain-submissions/new")
- now = datetime.now(timezone.utc)
- seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
-
- # How to pad a string with zeros: https://stackoverflow.com/a/39402910
- # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough
- padded_seconds = str(seconds).zfill(5)
-
- # See discussion here: https://stackoverflow.com/a/13484764
- uid = str(uuid4())[:8]
- filename = f'1/{VERSION}/{now.date()}/1/{user_id_hash}/{padded_seconds}__{uid}.json.gz'
-
# Using an approach from https://stackoverflow.com/a/30476450
+ now = datetime.now(timezone.utc)
epoch_time = (now - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
hashed_batch = HashedBatch(user_id_hash=user_id_hash, timestamp=epoch_time, items=batch.items)
stats_manager.record_batch(hashed_batch)
- data = gzip.compress(hashed_batch.json().encode('utf8'))
- upload(data, filename)
+ filename = upload_object(hashed_batch, now, user_id_hash, "batch")
global last_batch
last_batch = hashed_batch
@@ -118,6 +111,21 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'url': batch_url,
}
+ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):
+ seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
+
+ # How to pad a string with zeros: https://stackoverflow.com/a/39402910
+ # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough
+ padded_seconds = str(seconds).zfill(5)
+
+ # See discussion here: https://stackoverflow.com/a/13484764
+ uid = str(uuid4())[:8]
+
+ filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'
+ data = gzip.compress(model_object.json().encode('utf8'))
+ upload(data, filename)
+ return filename
+
@router.post('/batches/new')
def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:
user_id_hash = _get_user_id_hash(batch_request)
@@ -163,6 +171,25 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'status': 'ok'
}
+ @router.post('/results')
+ def post_results(request, results: Results):
+ # Check the API key
+ api_key = ApiKey.objects.get(key=results.api_key)
+ if api_key is None:
+ raise HTTPException(401, "Invalid API key")
+
+ documents = [Document(url=result.url, title=result.title, extract=result.extract) for result in results.results]
+ index_path = f"{settings.DATA_PATH}/{settings.INDEX_NAME}"
+ index_documents(documents, index_path)
+
+ now = datetime.now(timezone.utc)
+ filename = upload_object(results, now, api_key.user.username, "results")
+
+ return {
+ 'status': 'ok',
+ 'url': f'{PUBLIC_URL_PREFIX}{filename}',
+ }
+
return router
diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.py
index 2380e49..433fa2e 100644
--- a/mwmbl/crawler/batch.py
+++ b/mwmbl/crawler/batch.py
@@ -58,3 +58,17 @@ class HashedBatch(Schema):
user_id_hash: str
timestamp: int
items: list[Item]
+
+
+class Result(Schema):
+ url: str
+ title: str
+ extract: str
+ score: Optional[float]
+ term: Optional[str]
+ state: Optional[int]
+
+
+class Results(Schema):
+ api_key: str
+ results: list[Result]
diff --git a/mwmbl/database.py b/mwmbl/database.py
index c55cd11..10d874b 100644
--- a/mwmbl/database.py
+++ b/mwmbl/database.py
@@ -1,6 +1,7 @@
-from django.conf import settings
from psycopg2 import connect
+from mwmbl import settings
+
class Database:
def __init__(self):
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index c221ade..480856a 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -46,11 +46,16 @@ def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Count
start_time = datetime.utcnow()
document_tuples = list(get_documents_from_batches(batch_data))
documents = [Document(title, url, extract) for title, url, extract in document_tuples]
+ end_time, new_page_doc_counts = index_documents(documents, index_path)
+ logger.info(f"Indexing took {end_time - start_time}")
+ return new_page_doc_counts
+
+
+def index_documents(documents, index_path):
page_documents = preprocess_documents(documents, index_path)
new_page_doc_counts = index_pages(index_path, page_documents)
end_time = datetime.utcnow()
- logger.info(f"Indexing took {end_time - start_time}")
- return new_page_doc_counts
+ return end_time, new_page_doc_counts
def index_pages(index_path: str, page_documents: dict[int, list[Document]]) -> Counter:
diff --git a/mwmbl/migrations/0010_apikey.py b/mwmbl/migrations/0010_apikey.py
new file mode 100644
index 0000000..505bb5d
--- /dev/null
+++ b/mwmbl/migrations/0010_apikey.py
@@ -0,0 +1,25 @@
+# Generated by Django 4.2.14 on 2024-08-09 18:21
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import mwmbl.models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('mwmbl', '0009_alter_domainsubmission_options'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='ApiKey',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('key', models.CharField(default=mwmbl.models.random_api_key, max_length=300, unique=True)),
+ ('created_on', models.DateTimeField()),
+ ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+ ],
+ ),
+ ]
diff --git a/mwmbl/models.py b/mwmbl/models.py
index e996791..91026dc 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -1,3 +1,5 @@
+import secrets
+
from django.db import models
from django.contrib.auth.models import AbstractUser
@@ -93,3 +95,13 @@ class DomainSubmission(models.Model):
status_changed_on = models.DateTimeField(null=True, blank=True)
rejection_reason = models.CharField(max_length=20, choices=[(k, v) for k, v in DOMAIN_REJECTION_REASON.items()], blank=True)
rejection_detail = models.CharField(max_length=300, blank=True)
+
+
+def random_api_key():
+ return secrets.token_urlsafe(64)
+
+
+class ApiKey(models.Model):
+ user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)
+ key = models.CharField(max_length=300, unique=True, default=random_api_key)
+ created_on = models.DateTimeField()
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 2540eec..b3e4605 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -20,9 +20,11 @@ from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
from mwmbl.tokenizer import tokenize, get_bigrams
from mwmbl.utils import request_cache
+
logger = getLogger(__name__)
+MAX_QUERY_CHARS = 100
MATCH_SCORE_THRESHOLD = 0.0
SCORE_THRESHOLD = 0.0
LENGTH_PENALTY = 0.04
@@ -340,10 +342,17 @@ def clean_html(s: str):
return html.unescape(HTML_TAG_REGEX.sub('', s))
-def get_wiki_results(s: str, max_wiki_results: int):
+def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:
escaped_query = urllib.parse.quote(s, safe='')
with request_cache(timedelta(weeks=10)) as session:
wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()
+
+ if 'query' not in wiki_response or 'search' not in wiki_response['query']:
+ if 'error' in wiki_response:
+ logger.warning("Error in wiki response: %s", wiki_response['error'])
+
+ return []
+
wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
max_wiki_results + 1 - i, s, state=DocumentState.FROM_WIKI)
for i, result in enumerate(wiki_response['query']['search'][:max_wiki_results])]
@@ -364,12 +373,14 @@ class HeuristicAndWikiRanker(HeuristicRanker):
self.max_wiki_results = max_wiki_results
def search(self, s: str, additional_results: list[Document]) -> list[Document]:
- results = super().search(s, additional_results)
+ s_shortened = s[:MAX_QUERY_CHARS]
+
+ results = super().search(s_shortened, additional_results)
if len(results) == 0 and self.return_none_if_no_mwmbl_results:
return []
- wiki_results = get_wiki_results(s, self.max_wiki_results)
+ wiki_results = get_wiki_results(s_shortened, self.max_wiki_results)
return wiki_results + results
diff --git a/mwmbl/views.py b/mwmbl/views.py
index 5107ed3..88ff881 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -64,10 +64,6 @@ def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,
return paragraphs, title
-class Optionallist:
- pass
-
-
def _prepare_results(results: Optional[list[Document]]) -> Optional[dict[str, list[Document]]]:
if results is None:
return None