Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Make justext part of mwmbl to make building easier

Daoud Clarkeccd457a331ef
diff --git a/Dockerfile b/Dockerfile
index 378123f..12debb5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,7 +28,6 @@ RUN python -m venv /venv
# Copy only the necessary files to build/install the python package
COPY pyproject.toml poetry.lock /app/
COPY mwmbl /app/mwmbl
-COPY justext /app/justext
# Working directory is /app
# Use pip to install the mwmbl python package
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 303fcfb..4c3e05f 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -1,5 +1,4 @@
import re
-import re
import time
from logging import getLogger
from multiprocessing.pool import ThreadPool
@@ -8,12 +7,13 @@ from urllib.parse import urlparse, urlunsplit, urljoin
from urllib.robotparser import RobotFileParser
import requests
-from justext import core, utils
-from justext.core import html_to_dom
-from justext.paragraph import Paragraph
+from mwmbl.justext import core, utils
from requests import ReadTimeout
from urllib3.exceptions import NewConnectionError, MaxRetryError
+from mwmbl.justext.core import html_to_dom
+from mwmbl.justext.paragraph import Paragraph
+
ALLOWED_EXCEPTIONS = (ValueError, ConnectionError, ReadTimeout, TimeoutError,
OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)
diff --git a/justext/__init__.py b/mwmbl/justext/__init__.py
similarity index 100%
rename from justext/__init__.py
rename to mwmbl/justext/__init__.py
diff --git a/justext/core.py b/mwmbl/justext/core.py
similarity index 100%
rename from justext/core.py
rename to mwmbl/justext/core.py
diff --git a/justext/paragraph.py b/mwmbl/justext/paragraph.py
similarity index 100%
rename from justext/paragraph.py
rename to mwmbl/justext/paragraph.py
diff --git a/justext/stoplists/English.txt b/mwmbl/justext/stoplists/English.txt
similarity index 100%
rename from justext/stoplists/English.txt
rename to mwmbl/justext/stoplists/English.txt
diff --git a/justext/utils.py b/mwmbl/justext/utils.py
similarity index 100%
rename from justext/utils.py
rename to mwmbl/justext/utils.py
diff --git a/mwmbl/views.py b/mwmbl/views.py
index 88ff881..ef2bf55 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -1,24 +1,23 @@
-import re
from collections import defaultdict
from dataclasses import asdict
from datetime import datetime
from logging import getLogger
-from typing import Optional, Any
+from typing import Optional
from urllib.parse import urlencode
-import justext
+from mwmbl import justext
import objgraph
import psutil
import requests
from django.conf import settings
from django.contrib.auth.decorators import login_required, permission_required
from django.contrib.auth.mixins import LoginRequiredMixin
-from django.forms import ModelForm, ModelChoiceField, RadioSelect, CharField
+from django.forms import ModelForm, RadioSelect, CharField
from django.http import HttpResponseBadRequest
from django.shortcuts import render, redirect
from django.views.decorators.http import require_http_methods
from django.views.generic import DetailView, ListView
-from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revise_paragraph_classification, \
+from mwmbl.justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revise_paragraph_classification, \
LENGTH_LOW_DEFAULT, STOPWORDS_LOW_DEFAULT, MAX_LINK_DENSITY_DEFAULT, NO_HEADINGS_DEFAULT, LENGTH_HIGH_DEFAULT, \
STOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessor
from requests.exceptions import RequestException