An open source, non-profit web search engine
renamed package to mwmbl - renamed package to mwmbl in pyproject.toml - tinysearchengine and indexer modules have been moved into mwmbl package folder - analyse module has been left as is in the root of the repo - import statements in tinysearchengine now use mwmbl.tinysearchengine - import statements in indexer now use mwmbl.indexer or mwmbl.tinysearchengine or relative imports like .paths - import statements in analyse now use mwmbl.indexer or mwmbl.tinysearchengine - final CMD in Dockerfile now uses updated path mwmbl.tinysearchengine.app - fixed a couple of import statement errors in tinysearchengine/indexer.py
nitred11eedcde8428
diff --git a/Dockerfile b/Dockerfileindex 784381a..3f525b8 100644--- a/Dockerfile+++ b/Dockerfile@@ -31,4 +31,4 @@ COPY data /data#COPY docker-entrypoint.sh wsgi.py ./#CMD ["./docker-entrypoint.sh"]-CMD ["/venv/bin/python", "-m", "tinysearchengine.app", "/data/index.tinysearch"]+CMD ["/venv/bin/python", "-m", "mwmbl.tinysearchengine.app", "/data/index.tinysearch"]diff --git a/analyse/inspect_index.py b/analyse/inspect_index.pyindex f73064a..18a5a96 100644--- a/analyse/inspect_index.py+++ b/analyse/inspect_index.py@@ -1,5 +1,5 @@-from tinysearchengine.indexer import TinyIndex, NUM_PAGES, PAGE_SIZE, Document-from indexer.paths import INDEX_PATH+from mwmbl.tinysearchengine.indexer import TinyIndex, NUM_PAGES, PAGE_SIZE, Document+from mwmbl.indexer.paths import INDEX_PATHdef get_items():diff --git a/analyse/make_curl.py b/analyse/make_curl.pyindex c411806..465f990 100644--- a/analyse/make_curl.py+++ b/analyse/make_curl.py@@ -5,8 +5,8 @@ import osfrom itertools import islicefrom urllib.parse import quote-from indexer.paths import DATA_DIR-from indexer.wiki import get_wiki_titles_and_urls+from mwmbl.indexer.paths import DATA_DIR+from mwmbl.indexer.wiki import get_wiki_titles_and_urlsURL_TEMPLATE = "http://localhost:8000/complete?q={}"CURL_FILE = os.path.join(DATA_DIR, "urls.curl")diff --git a/analyse/performance.py b/analyse/performance.pyindex 53fdcae..4a675d4 100644--- a/analyse/performance.py+++ b/analyse/performance.py@@ -8,11 +8,11 @@ import numpy as npfrom spacy.lang.en import Englishfrom starlette.testclient import TestClient-from tinysearchengine import create_app-from indexer.fsqueue import ZstdJsonSerializer-from indexer.index import index_titles_urls_and_extracts-from tinysearchengine.indexer import TinyIndex, TinyIndexer, Document-from indexer.paths import TEST_INDEX_PATH, DATA_DIR, TEST_TERMS_PATH+from mwmbl.tinysearchengine import create_app+from mwmbl.indexer.fsqueue import ZstdJsonSerializer+from mwmbl.indexer.index import index_titles_urls_and_extracts+from mwmbl.tinysearchengine.indexer import TinyIndex, TinyIndexer, Document+from mwmbl.indexer.paths import TEST_INDEX_PATH, DATA_DIR, TEST_TERMS_PATHNUM_DOCUMENTS = 30000NUM_PAGES_FOR_STATS = 10diff --git a/indexer/__init__.py b/mwmbl/__init__.pysimilarity index 100%rename from indexer/__init__.pyrename to mwmbl/__init__.pydiff --git a/indexer/domains/__init__.py b/mwmbl/indexer/__init__.pysimilarity index 100%rename from indexer/domains/__init__.pyrename to mwmbl/indexer/__init__.pydiff --git a/indexer/bootstrap.sh b/mwmbl/indexer/bootstrap.shsimilarity index 100%rename from indexer/bootstrap.shrename to mwmbl/indexer/bootstrap.shdiff --git a/indexer/crawl.py b/mwmbl/indexer/crawl.pysimilarity index 94%rename from indexer/crawl.pyrename to mwmbl/indexer/crawl.pyindex 609deb6..11405d0 100644--- a/indexer/crawl.py+++ b/mwmbl/indexer/crawl.py@@ -10,7 +10,7 @@ from traceback import print_tb, print_excimport pandas as pdimport requests-from paths import DATA_DIR, HN_TOP_PATH, CRAWL_PREFIX+from .paths import DATA_DIR, HN_TOP_PATH, CRAWL_PREFIXdef crawl():diff --git a/indexer/deploy.sh b/mwmbl/indexer/deploy.shsimilarity index 100%rename from indexer/deploy.shrename to mwmbl/indexer/deploy.shdiff --git a/indexer/domains.py b/mwmbl/indexer/domains.pysimilarity index 100%rename from indexer/domains.pyrename to mwmbl/indexer/domains.pydiff --git a/tinysearchengine/__init__.py b/mwmbl/indexer/domains/__init__.pysimilarity index 100%rename from tinysearchengine/__init__.pyrename to mwmbl/indexer/domains/__init__.pydiff --git a/indexer/domains/domain_titles.py b/mwmbl/indexer/domains/domain_titles.pysimilarity index 94%rename from indexer/domains/domain_titles.pyrename to mwmbl/indexer/domains/domain_titles.pyindex be6203d..907367e 100644--- a/indexer/domains/domain_titles.py+++ b/mwmbl/indexer/domains/domain_titles.py@@ -8,8 +8,8 @@ from urllib.parse import urlsplit, urlunsplitimport bs4import requests-from indexer.fsqueue import FSQueue, ZstdJsonSerializer-from indexer.paths import DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAME+from mwmbl.indexer.fsqueue import FSQueue, ZstdJsonSerializer+from mwmbl.indexer.paths import DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAMENUM_PROCESSES = 10diff --git a/indexer/domains/queue_domains.py b/mwmbl/indexer/domains/queue_domains.pysimilarity index 82%rename from indexer/domains/queue_domains.pyrename to mwmbl/indexer/domains/queue_domains.pyindex 8136de2..3eb7ac6 100644--- a/indexer/domains/queue_domains.py+++ b/mwmbl/indexer/domains/queue_domains.py@@ -4,8 +4,8 @@ Add domains to the queue to be retrievedimport csvimport gzip-from indexer.fsqueue import FSQueue, ZstdJsonSerializer-from indexer.paths import DOMAINS_PATH, DOMAINS_QUEUE_NAME, DATA_DIR+from mwmbl.indexer.fsqueue import FSQueue, ZstdJsonSerializer+from mwmbl.indexer.paths import DOMAINS_PATH, DOMAINS_QUEUE_NAME, DATA_DIRBATCH_SIZE = 250diff --git a/indexer/extract.py b/mwmbl/indexer/extract.pysimilarity index 100%rename from indexer/extract.pyrename to mwmbl/indexer/extract.pydiff --git a/indexer/extract_local.py b/mwmbl/indexer/extract_local.pysimilarity index 92%rename from indexer/extract_local.pyrename to mwmbl/indexer/extract_local.pyindex 040883f..b293f08 100644--- a/indexer/extract_local.py+++ b/mwmbl/indexer/extract_local.py@@ -4,9 +4,9 @@ import osfrom glob import globfrom multiprocessing import Process, Lock-from extract_process import fetch_process_warc_records-from fsqueue import FSQueue, GzipJsonRowSerializer-from paths import DATA_DIR+from .extract_process import fetch_process_warc_records+from .fsqueue import FSQueue, GzipJsonRowSerializer+from .paths import DATA_DIRARCHIVE_INFO_GLOB = 'outputs/records/*.gz'diff --git a/indexer/extract_process.py b/mwmbl/indexer/extract_process.pysimilarity index 100%rename from indexer/extract_process.pyrename to mwmbl/indexer/extract_process.pydiff --git a/indexer/fsqueue.py b/mwmbl/indexer/fsqueue.pysimilarity index 100%rename from indexer/fsqueue.pyrename to mwmbl/indexer/fsqueue.pydiff --git a/indexer/hn-top-domains-filtered.py b/mwmbl/indexer/hn-top-domains-filtered.pysimilarity index 100%rename from indexer/hn-top-domains-filtered.pyrename to mwmbl/indexer/hn-top-domains-filtered.pydiff --git a/indexer/index.py b/mwmbl/indexer/index.pysimilarity index 96%rename from indexer/index.pyrename to mwmbl/indexer/index.pyindex 3350560..d0f0efe 100644--- a/indexer/index.py+++ b/mwmbl/indexer/index.py@@ -10,7 +10,7 @@ import pandas as pd# NUM_PAGES = 8192# PAGE_SIZE = 512-from tinysearchengine.indexer import TinyIndexer, Document, TokenizedDocument+from mwmbl.tinysearchengine.indexer import TinyIndexer, Document, TokenizedDocumentNUM_INITIAL_TOKENS = 50diff --git a/indexer/index_glob.py b/mwmbl/indexer/index_glob.pysimilarity index 83%rename from indexer/index_glob.pyrename to mwmbl/indexer/index_glob.pyindex 31decc3..e9102c2 100644--- a/indexer/index_glob.py+++ b/mwmbl/indexer/index_glob.py@@ -4,12 +4,13 @@ from glob import globimport bs4from spacy.lang.en import English-from index import tokenize-from tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE-from paths import INDEX_PATH, CRAWL_GLOB+from .index import tokenize+from mwmbl.tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE+from .paths import INDEX_PATH, CRAWL_GLOBdef run():+ # TODO: item_factory argument is unfilled.indexer = TinyIndexer(INDEX_PATH, NUM_PAGES, PAGE_SIZE)indexer.create_if_not_exists()nlp = English()diff --git a/indexer/index_queue.py b/mwmbl/indexer/index_queue.pysimilarity index 74%rename from indexer/index_queue.pyrename to mwmbl/indexer/index_queue.pyindex eadfd75..f048e28 100644--- a/indexer/index_queue.py+++ b/mwmbl/indexer/index_queue.py@@ -3,10 +3,10 @@ Index items in the file-system queue"""from spacy.lang.en import English-from fsqueue import FSQueue, ZstdJsonSerializer-from index import index_titles_urls_and_extracts-from tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE-from paths import DATA_DIR, DOMAINS_TITLES_QUEUE_NAME, INDEX_PATH+from .fsqueue import FSQueue, ZstdJsonSerializer+from .index import index_titles_urls_and_extracts+from mwmbl.tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE+from .paths import DATA_DIR, DOMAINS_TITLES_QUEUE_NAME, INDEX_PATHdef get_queue_items():diff --git a/indexer/indexcc.py b/mwmbl/indexer/indexcc.pysimilarity index 82%rename from indexer/indexcc.pyrename to mwmbl/indexer/indexcc.pyindex 549bb0e..4f68025 100644--- a/indexer/indexcc.py+++ b/mwmbl/indexer/indexcc.py@@ -7,10 +7,10 @@ from logging import getLoggerimport spacy-from fsqueue import FSQueue, GzipJsonRowSerializer, FSQueueError-from index import index_titles_urls_and_extracts-from tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE, Document-from paths import INDEX_PATH, DATA_DIR, COMMON_CRAWL_TERMS_PATH+from .fsqueue import FSQueue, GzipJsonRowSerializer, FSQueueError+from .index import index_titles_urls_and_extracts+from mwmbl.tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE, Document+from .paths import INDEX_PATH, DATA_DIR, COMMON_CRAWL_TERMS_PATHlogging.basicConfig(level=logging.DEBUG, stream=sys.stdout)diff --git a/indexer/paths.py b/mwmbl/indexer/paths.pysimilarity index 100%rename from indexer/paths.pyrename to mwmbl/indexer/paths.pydiff --git a/indexer/wiki.py b/mwmbl/indexer/wiki.pysimilarity index 85%rename from indexer/wiki.pyrename to mwmbl/indexer/wiki.pyindex a3a66ff..93ac1c7 100644--- a/indexer/wiki.py+++ b/mwmbl/indexer/wiki.py@@ -7,9 +7,9 @@ from urllib.parse import quotefrom spacy.lang.en import English-from indexer.index import index_titles_urls_and_extracts-from tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE-from indexer.paths import WIKI_TITLES_PATH, INDEX_PATH+from .index import index_titles_urls_and_extracts+from mwmbl.tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE+from .paths import WIKI_TITLES_PATH, INDEX_PATHTEXT_TAGS = ['mediawiki', 'page', 'revision', 'text']TITLE_START = '<title>Wikipedia: 'diff --git a/mwmbl/tinysearchengine/__init__.py b/mwmbl/tinysearchengine/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/mwmbl/tinysearchengine/app.py b/mwmbl/tinysearchengine/app.pynew file mode 100644index 0000000..daaca87--- /dev/null+++ b/mwmbl/tinysearchengine/app.py@@ -0,0 +1,17 @@+import logging+import sys++import uvicorn++from mwmbl.tinysearchengine import create_app+from mwmbl.tinysearchengine.indexer import TinyIndex, NUM_PAGES, PAGE_SIZE, Document++logging.basicConfig()+++index_path = sys.argv[1]+tiny_index = TinyIndex(Document, index_path, NUM_PAGES, PAGE_SIZE)+app = create_app.create(tiny_index)++if __name__ == "__main__":+ uvicorn.run("mwmbl.tinysearchengine.app:app", host="0.0.0.0", port=8080, log_level="info")diff --git a/tinysearchengine/create_app.py b/mwmbl/tinysearchengine/create_app.pysimilarity index 98%rename from tinysearchengine/create_app.pyrename to mwmbl/tinysearchengine/create_app.pyindex 19c4a3b..7af8e9f 100644--- a/tinysearchengine/create_app.py+++ b/mwmbl/tinysearchengine/create_app.py@@ -7,7 +7,7 @@ from fastapi import FastAPIfrom starlette.responses import FileResponsefrom starlette.staticfiles import StaticFiles-from tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.indexer import TinyIndex, Documentlogger = getLogger(__name__)diff --git a/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pysimilarity index 98%rename from tinysearchengine/indexer.pyrename to mwmbl/tinysearchengine/indexer.pyindex 7e17dc1..d5fe684 100644--- a/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -6,7 +6,7 @@ from pathlib import Pathfrom typing import TypeVar, Generic, Callable, Listimport mmh3-from zstandard import ZstdDecompressor+from zstandard import ZstdDecompressor, ZstdCompressor, ZstdErrorNUM_PAGES = 25600diff --git a/tinysearchengine/static/index.css b/mwmbl/tinysearchengine/static/index.csssimilarity index 100%rename from tinysearchengine/static/index.cssrename to mwmbl/tinysearchengine/static/index.cssdiff --git a/tinysearchengine/static/index.html b/mwmbl/tinysearchengine/static/index.htmlsimilarity index 100%rename from tinysearchengine/static/index.htmlrename to mwmbl/tinysearchengine/static/index.htmldiff --git a/tinysearchengine/static/index.js b/mwmbl/tinysearchengine/static/index.jssimilarity index 100%rename from tinysearchengine/static/index.jsrename to mwmbl/tinysearchengine/static/index.jsdiff --git a/tinysearchengine/static/landing.html b/mwmbl/tinysearchengine/static/landing.htmlsimilarity index 100%rename from tinysearchengine/static/landing.htmlrename to mwmbl/tinysearchengine/static/landing.htmldiff --git a/tinysearchengine/static/plugin.xml b/mwmbl/tinysearchengine/static/plugin.xmlsimilarity index 100%rename from tinysearchengine/static/plugin.xmlrename to mwmbl/tinysearchengine/static/plugin.xmldiff --git a/tinysearchengine/static/search.html b/mwmbl/tinysearchengine/static/search.htmlsimilarity index 100%rename from tinysearchengine/static/search.htmlrename to mwmbl/tinysearchengine/static/search.htmldiff --git a/tinysearchengine/static/typeahead.css b/mwmbl/tinysearchengine/static/typeahead.csssimilarity index 100%rename from tinysearchengine/static/typeahead.cssrename to mwmbl/tinysearchengine/static/typeahead.cssdiff --git a/tinysearchengine/static/typeahead.js b/mwmbl/tinysearchengine/static/typeahead.jssimilarity index 100%rename from tinysearchengine/static/typeahead.jsrename to mwmbl/tinysearchengine/static/typeahead.jsdiff --git a/pyproject.toml b/pyproject.tomlindex c634fc9..c51ea16 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,5 +1,5 @@[tool.poetry]-name = "tinysearchengine"+name = "mwmbl"version = "0.1.0"description = ""authors = ["Daoud Clarke <[email protected]>"]diff --git a/tinysearchengine/app.py b/tinysearchengine/app.pydeleted file mode 100644index 27e8946..0000000--- a/tinysearchengine/app.py+++ /dev/null@@ -1,17 +0,0 @@-import logging-import sys--import uvicorn--from tinysearchengine import create_app-from tinysearchengine.indexer import TinyIndex, NUM_PAGES, PAGE_SIZE, Document--logging.basicConfig()---index_path = sys.argv[1]-tiny_index = TinyIndex(Document, index_path, NUM_PAGES, PAGE_SIZE)-app = create_app.create(tiny_index)--if __name__ == "__main__":- uvicorn.run("tinysearchengine.app:app", host="0.0.0.0", port=8080, log_level="info")