An open source, non-profit web search engine
Move indexer code to a separate package
Daoud Clarkebaede32298b7
diff --git a/analyse/inspect_index.py b/analyse/inspect_index.pyindex 4a0e530..f73064a 100644--- a/analyse/inspect_index.py+++ b/analyse/inspect_index.py@@ -1,5 +1,5 @@from tinysearchengine.indexer import TinyIndex, NUM_PAGES, PAGE_SIZE, Document-from paths import INDEX_PATH+from indexer.paths import INDEX_PATHdef get_items():diff --git a/make_curl.py b/analyse/make_curl.pysimilarity index 87%rename from make_curl.pyrename to analyse/make_curl.pyindex f7e7ed5..c411806 100644--- a/make_curl.py+++ b/analyse/make_curl.py@@ -5,8 +5,8 @@ import osfrom itertools import islicefrom urllib.parse import quote-from paths import DATA_DIR-from wiki import get_wiki_titles_and_urls+from indexer.paths import DATA_DIR+from indexer.wiki import get_wiki_titles_and_urlsURL_TEMPLATE = "http://localhost:8000/complete?q={}"CURL_FILE = os.path.join(DATA_DIR, "urls.curl")diff --git a/performance.py b/analyse/performance.pysimilarity index 95%rename from performance.pyrename to analyse/performance.pyindex b8a2496..53fdcae 100644--- a/performance.py+++ b/analyse/performance.py@@ -9,10 +9,10 @@ from spacy.lang.en import Englishfrom starlette.testclient import TestClientfrom tinysearchengine import create_app-from fsqueue import ZstdJsonSerializer-from index import index_titles_urls_and_extracts+from indexer.fsqueue import ZstdJsonSerializer+from indexer.index import index_titles_urls_and_extractsfrom tinysearchengine.indexer import TinyIndex, TinyIndexer, Document-from paths import TEST_INDEX_PATH, DATA_DIR, TEST_TERMS_PATH+from indexer.paths import TEST_INDEX_PATH, DATA_DIR, TEST_TERMS_PATHNUM_DOCUMENTS = 30000NUM_PAGES_FOR_STATS = 10diff --git a/domains/__init__.py b/indexer/__init__.pysimilarity index 100%rename from domains/__init__.pyrename to indexer/__init__.pydiff --git a/bootstrap.sh b/indexer/bootstrap.shsimilarity index 100%rename from bootstrap.shrename to indexer/bootstrap.shdiff --git a/crawl.py b/indexer/crawl.pysimilarity index 100%rename from crawl.pyrename to indexer/crawl.pydiff --git a/deploy.sh b/indexer/deploy.shsimilarity index 100%rename from deploy.shrename to indexer/deploy.shdiff --git a/domains.py b/indexer/domains.pysimilarity index 100%rename from domains.pyrename to indexer/domains.pydiff --git a/indexer/domains/__init__.py b/indexer/domains/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/domains/domain_titles.py b/indexer/domains/domain_titles.pysimilarity index 95%rename from domains/domain_titles.pyrename to indexer/domains/domain_titles.pyindex 62232b3..be6203d 100644--- a/domains/domain_titles.py+++ b/indexer/domains/domain_titles.py@@ -8,8 +8,8 @@ from urllib.parse import urlsplit, urlunsplitimport bs4import requests-from fsqueue import FSQueue, ZstdJsonSerializer-from paths import DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAME+from indexer.fsqueue import FSQueue, ZstdJsonSerializer+from indexer.paths import DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAMENUM_PROCESSES = 10diff --git a/domains/queue_domains.py b/indexer/domains/queue_domains.pysimilarity index 83%rename from domains/queue_domains.pyrename to indexer/domains/queue_domains.pyindex 983b6c2..8136de2 100644--- a/domains/queue_domains.py+++ b/indexer/domains/queue_domains.py@@ -4,8 +4,8 @@ Add domains to the queue to be retrievedimport csvimport gzip-from fsqueue import FSQueue, ZstdJsonSerializer-from paths import DOMAINS_PATH, DOMAINS_QUEUE_NAME, DATA_DIR+from indexer.fsqueue import FSQueue, ZstdJsonSerializer+from indexer.paths import DOMAINS_PATH, DOMAINS_QUEUE_NAME, DATA_DIRBATCH_SIZE = 250diff --git a/extract.py b/indexer/extract.pysimilarity index 100%rename from extract.pyrename to indexer/extract.pydiff --git a/extract_local.py b/indexer/extract_local.pysimilarity index 100%rename from extract_local.pyrename to indexer/extract_local.pydiff --git a/extract_process.py b/indexer/extract_process.pysimilarity index 100%rename from extract_process.pyrename to indexer/extract_process.pydiff --git a/fsqueue.py b/indexer/fsqueue.pysimilarity index 100%rename from fsqueue.pyrename to indexer/fsqueue.pydiff --git a/hn-top-domains-filtered.py b/indexer/hn-top-domains-filtered.pysimilarity index 100%rename from hn-top-domains-filtered.pyrename to indexer/hn-top-domains-filtered.pydiff --git a/index.py b/indexer/index.pysimilarity index 100%rename from index.pyrename to indexer/index.pydiff --git a/index_glob.py b/indexer/index_glob.pysimilarity index 100%rename from index_glob.pyrename to indexer/index_glob.pydiff --git a/index_queue.py b/indexer/index_queue.pysimilarity index 100%rename from index_queue.pyrename to indexer/index_queue.pydiff --git a/indexcc.py b/indexer/indexcc.pysimilarity index 100%rename from indexcc.pyrename to indexer/indexcc.pydiff --git a/paths.py b/indexer/paths.pysimilarity index 100%rename from paths.pyrename to indexer/paths.pydiff --git a/wiki.py b/indexer/wiki.pysimilarity index 91%rename from wiki.pyrename to indexer/wiki.pyindex 9f4e414..a3a66ff 100644--- a/wiki.py+++ b/indexer/wiki.py@@ -7,9 +7,9 @@ from urllib.parse import quotefrom spacy.lang.en import English-from index import index_titles_urls_and_extracts+from indexer.index import index_titles_urls_and_extractsfrom tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE-from paths import WIKI_TITLES_PATH, INDEX_PATH+from indexer.paths import WIKI_TITLES_PATH, INDEX_PATHTEXT_TAGS = ['mediawiki', 'page', 'revision', 'text']TITLE_START = '<title>Wikipedia: '