An open source, non-profit web search engine
Add migrations; remove unused code
Daoud Clarked7dbb9fda673
diff --git a/mwmbl/migrations/0002_curation.py b/mwmbl/migrations/0002_curation.pynew file mode 100644index 0000000..17a81fb--- /dev/null+++ b/mwmbl/migrations/0002_curation.py@@ -0,0 +1,27 @@+# Generated by Django 4.2.6 on 2023-12-02 12:42++from django.conf import settings+from django.db import migrations, models+import django.db.models.deletion+++class Migration(migrations.Migration):++ dependencies = [+ ('mwmbl', '0001_initial'),+ ]++ operations = [+ migrations.CreateModel(+ name='Curation',+ fields=[+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),+ ('timestamp', models.DateTimeField()),+ ('query', models.CharField(max_length=300)),+ ('original_results', models.JSONField()),+ ('new_results', models.JSONField()),+ ('num_changes', models.IntegerField(default=0)),+ ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ],+ ),+ ]diff --git a/mwmbl/migrations/0003_auto_20231203_0614.py b/mwmbl/migrations/0003_auto_20231203_0614.pynew file mode 100644index 0000000..7a4be6d--- /dev/null+++ b/mwmbl/migrations/0003_auto_20231203_0614.py@@ -0,0 +1,55 @@+# Generated by Django 4.2.6 on 2023-12-03 06:14+from itertools import groupby+from urllib.parse import urlparse, parse_qs++from django.db import migrations+++def create_curations_from_user_curation(apps, schema_editor):+ Curation = apps.get_model('mwmbl', 'Curation')+ UserCuration = apps.get_model('mwmbl', 'UserCuration')++ # Order curations by timestamp+ curations = UserCuration.objects.all().order_by('timestamp')++ # Group by user and url+ for (user, url), curations in groupby(curations, lambda c: (c.user, c.url)):+ curations = list(curations)++ # Get the original results from the first curation+ original_results = curations[0].results++ # Get the new results from the last curation+ new_results = curations[-1].results++ # Get the number of changes from the number of curations+ num_changes = len(curations)++ # Get the timestamp from the first curation+ timestamp = curations[0].timestamp++ # Get the query from the url+ parsed_url_query = parse_qs(urlparse(url).query)+ query = parsed_url_query.get("q", [""])[0]++ # Create the new curation+ curation = Curation(+ timestamp=timestamp,+ query=query,+ original_results=original_results,+ new_results=new_results,+ num_changes=num_changes,+ user=user,+ )+ curation.save()+++class Migration(migrations.Migration):++ dependencies = [+ ('mwmbl', '0002_curation'),+ ]++ operations = [+ migrations.RunPython(create_curations_from_user_curation),+ ]diff --git a/mwmbl/platform/curate.py b/mwmbl/platform/curate.pydeleted file mode 100644index 95049d8..0000000--- a/mwmbl/platform/curate.py+++ /dev/null@@ -1,110 +0,0 @@-from logging import getLogger-from typing import Any, Optional-from urllib.parse import parse_qs--from ninja import Router, NinjaAPI--from mwmbl.indexer.update_urls import get_datetime_from_timestamp-from mwmbl.models import UserCuration-from mwmbl.platform.data import CurateBegin, CurateMove, CurateDelete, CurateAdd, CurateValidate, \- make_curation_type-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState-from mwmbl.tokenizer import tokenize-from mwmbl.utils import add_term_info, add_term_infos--RESULT_URL = "https://mwmbl.org/?q="-MAX_CURATED_SCORE = 1_111_111.0---logger = getLogger(__name__)---def create_router(index_path: str, version: str) -> NinjaAPI:- router = NinjaAPI(urls_namespace=f"curate-{version}", csrf=True)-- @router.post("/begin")- def user_begin_curate(request, curate_begin: make_curation_type(CurateBegin)):- return _curate(request, "curate_begin", curate_begin)-- @router.post("/move")- def user_move_result(request, curate_move: make_curation_type(CurateMove)):- return _curate(request, "curate_move", curate_move)-- @router.post("/delete")- def user_delete_result(request, curate_delete: make_curation_type(CurateDelete)):- return _curate(request, "curate_delete", curate_delete)-- @router.post("/add")- def user_add_result(request, curate_add: make_curation_type(CurateAdd)):- return _curate(request, "curate_add", curate_add)-- @router.post("/validate")- def user_add_result(request, curate_validate: make_curation_type(CurateValidate)):- return _curate(request, "curate_validate", curate_validate)-- def _curate(request, curation_type: str, curation: Any):- user_curation = UserCuration(- user=request.user,- timestamp=get_datetime_from_timestamp(curation.timestamp / 1000.0),- url=curation.url,- results=curation.dict()["results"],- curation_type=curation_type,- curation=curation.curation.dict(),- )- user_curation.save()-- with TinyIndex(Document, index_path, 'w') as indexer:- query_string = parse_qs(curation.url)- if len(query_string) > 1:- raise ValueError(f"Should be one query string in the URL: {curation.url}")-- queries = next(iter(query_string.values()))- if len(queries) > 1:- raise ValueError(f"Should be one query value in the URL: {curation.url}")-- query = queries[0]- tokens = tokenize(query)- term = " ".join(tokens)-- documents = [- Document(- title=result.title,- url=result.url,- extract=result.extract,- score=MAX_CURATED_SCORE - i,- term=term,- state=_get_document_state(result.validated, result.source),- )- for i, result in enumerate(curation.results)- ]-- page_index = indexer.get_key_page_index(term)- existing_documents_no_terms = indexer.get_page(page_index)- existing_documents = add_term_infos(existing_documents_no_terms, indexer, page_index)- other_documents = [doc for doc in existing_documents if doc.term != term]- logger.info(f"Found {len(other_documents)} other documents for term {term} at page {page_index} "- f"with terms { {doc.term for doc in other_documents} }")-- all_documents = documents + other_documents- logger.info(f"Storing {len(all_documents)} documents at page {page_index}")- indexer.store_in_page(page_index, all_documents)-- return {"curation": "ok"}-- return router---def _get_document_state(validated: bool, source: str) -> Optional[DocumentState]:- if validated:- if source.lower() == "user":- return DocumentState.FROM_USER_APPROVED- elif source.lower() == "google":- return DocumentState.FROM_GOOGLE_APPROVED- else:- return DocumentState.ORGANIC_APPROVED- elif source.lower() == "user":- return DocumentState.FROM_USER- elif source.lower() == "google":- return DocumentState.FROM_GOOGLE- else:- return Nonediff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex bd7a764..0f62bc3 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -18,8 +18,7 @@ from django.contrib import adminfrom django.urls import path, includeimport mwmbl.crawler.app as crawler-from mwmbl.platform import curate-from mwmbl.search_setup import queued_batches, index_path, ranker, batch_cache+from mwmbl.search_setup import queued_batches, ranker, batch_cachefrom mwmbl.tinysearchengine import searchfrom mwmbl.views import home_fragment, fetch_url, index, approve@@ -35,10 +34,8 @@ urlpatterns = [# TODO: this is the old API, deprecated and to be removed once all clients have moved overpath("search/", search.create_router(ranker, "0.1").urls),path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),- path("curation/", curate.create_router(index_path, version="0.1").urls),# New APIpath("api/v1/search/", search.create_router(ranker, "1.0.0").urls),path("api/v1/crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="1.0.0").urls),- path("api/v1/curation/", curate.create_router(index_path, version="1.0.0").urls),]