An open source, non-profit web search engine
Make justext part of mwmbl to make building easier
Daoud Clarkeccd457a331ef
diff --git a/Dockerfile b/Dockerfileindex 378123f..12debb5 100644--- a/Dockerfile+++ b/Dockerfile@@ -28,7 +28,6 @@ RUN python -m venv /venv# Copy only the necessary files to build/install the python packageCOPY pyproject.toml poetry.lock /app/COPY mwmbl /app/mwmbl-COPY justext /app/justext# Working directory is /app# Use pip to install the mwmbl python packagediff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 303fcfb..4c3e05f 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -1,5 +1,4 @@import re-import reimport timefrom logging import getLoggerfrom multiprocessing.pool import ThreadPool@@ -8,12 +7,13 @@ from urllib.parse import urlparse, urlunsplit, urljoinfrom urllib.robotparser import RobotFileParserimport requests-from justext import core, utils-from justext.core import html_to_dom-from justext.paragraph import Paragraph+from mwmbl.justext import core, utilsfrom requests import ReadTimeoutfrom urllib3.exceptions import NewConnectionError, MaxRetryError+from mwmbl.justext.core import html_to_dom+from mwmbl.justext.paragraph import Paragraph+ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)diff --git a/justext/__init__.py b/mwmbl/justext/__init__.pysimilarity index 100%rename from justext/__init__.pyrename to mwmbl/justext/__init__.pydiff --git a/justext/core.py b/mwmbl/justext/core.pysimilarity index 100%rename from justext/core.pyrename to mwmbl/justext/core.pydiff --git a/justext/paragraph.py b/mwmbl/justext/paragraph.pysimilarity index 100%rename from justext/paragraph.pyrename to mwmbl/justext/paragraph.pydiff --git a/justext/stoplists/English.txt b/mwmbl/justext/stoplists/English.txtsimilarity index 100%rename from justext/stoplists/English.txtrename to mwmbl/justext/stoplists/English.txtdiff --git a/justext/utils.py b/mwmbl/justext/utils.pysimilarity index 100%rename from justext/utils.pyrename to mwmbl/justext/utils.pydiff --git a/mwmbl/views.py b/mwmbl/views.pyindex 88ff881..ef2bf55 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -1,24 +1,23 @@-import refrom collections import defaultdictfrom dataclasses import asdictfrom datetime import datetimefrom logging import getLogger-from typing import Optional, Any+from typing import Optionalfrom urllib.parse import urlencode-import justext+from mwmbl import justextimport objgraphimport psutilimport requestsfrom django.conf import settingsfrom django.contrib.auth.decorators import login_required, permission_requiredfrom django.contrib.auth.mixins import LoginRequiredMixin-from django.forms import ModelForm, ModelChoiceField, RadioSelect, CharField+from django.forms import ModelForm, RadioSelect, CharFieldfrom django.http import HttpResponseBadRequestfrom django.shortcuts import render, redirectfrom django.views.decorators.http import require_http_methodsfrom django.views.generic import DetailView, ListView-from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revise_paragraph_classification, \+from mwmbl.justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revise_paragraph_classification, \LENGTH_LOW_DEFAULT, STOPWORDS_LOW_DEFAULT, MAX_LINK_DENSITY_DEFAULT, NO_HEADINGS_DEFAULT, LENGTH_HIGH_DEFAULT, \STOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessorfrom requests.exceptions import RequestException