Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Move indexer code to a separate package

Daoud Clarkebaede32298b7
diff --git a/analyse/inspect_index.py b/analyse/inspect_index.py
index 4a0e530..f73064a 100644
--- a/analyse/inspect_index.py
+++ b/analyse/inspect_index.py
@@ -1,5 +1,5 @@
from tinysearchengine.indexer import TinyIndex, NUM_PAGES, PAGE_SIZE, Document
-from paths import INDEX_PATH
+from indexer.paths import INDEX_PATH
def get_items():
diff --git a/make_curl.py b/analyse/make_curl.py
similarity index 87%
rename from make_curl.py
rename to analyse/make_curl.py
index f7e7ed5..c411806 100644
--- a/make_curl.py
+++ b/analyse/make_curl.py
@@ -5,8 +5,8 @@ import os
from itertools import islice
from urllib.parse import quote
-from paths import DATA_DIR
-from wiki import get_wiki_titles_and_urls
+from indexer.paths import DATA_DIR
+from indexer.wiki import get_wiki_titles_and_urls
URL_TEMPLATE = "http://localhost:8000/complete?q={}"
CURL_FILE = os.path.join(DATA_DIR, "urls.curl")
diff --git a/performance.py b/analyse/performance.py
similarity index 95%
rename from performance.py
rename to analyse/performance.py
index b8a2496..53fdcae 100644
--- a/performance.py
+++ b/analyse/performance.py
@@ -9,10 +9,10 @@ from spacy.lang.en import English
from starlette.testclient import TestClient
from tinysearchengine import create_app
-from fsqueue import ZstdJsonSerializer
-from index import index_titles_urls_and_extracts
+from indexer.fsqueue import ZstdJsonSerializer
+from indexer.index import index_titles_urls_and_extracts
from tinysearchengine.indexer import TinyIndex, TinyIndexer, Document
-from paths import TEST_INDEX_PATH, DATA_DIR, TEST_TERMS_PATH
+from indexer.paths import TEST_INDEX_PATH, DATA_DIR, TEST_TERMS_PATH
NUM_DOCUMENTS = 30000
NUM_PAGES_FOR_STATS = 10
diff --git a/domains/__init__.py b/indexer/__init__.py
similarity index 100%
rename from domains/__init__.py
rename to indexer/__init__.py
diff --git a/bootstrap.sh b/indexer/bootstrap.sh
similarity index 100%
rename from bootstrap.sh
rename to indexer/bootstrap.sh
diff --git a/crawl.py b/indexer/crawl.py
similarity index 100%
rename from crawl.py
rename to indexer/crawl.py
diff --git a/deploy.sh b/indexer/deploy.sh
similarity index 100%
rename from deploy.sh
rename to indexer/deploy.sh
diff --git a/domains.py b/indexer/domains.py
similarity index 100%
rename from domains.py
rename to indexer/domains.py
diff --git a/indexer/domains/__init__.py b/indexer/domains/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/domains/domain_titles.py b/indexer/domains/domain_titles.py
similarity index 95%
rename from domains/domain_titles.py
rename to indexer/domains/domain_titles.py
index 62232b3..be6203d 100644
--- a/domains/domain_titles.py
+++ b/indexer/domains/domain_titles.py
@@ -8,8 +8,8 @@ from urllib.parse import urlsplit, urlunsplit
import bs4
import requests
-from fsqueue import FSQueue, ZstdJsonSerializer
-from paths import DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAME
+from indexer.fsqueue import FSQueue, ZstdJsonSerializer
+from indexer.paths import DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAME
NUM_PROCESSES = 10
diff --git a/domains/queue_domains.py b/indexer/domains/queue_domains.py
similarity index 83%
rename from domains/queue_domains.py
rename to indexer/domains/queue_domains.py
index 983b6c2..8136de2 100644
--- a/domains/queue_domains.py
+++ b/indexer/domains/queue_domains.py
@@ -4,8 +4,8 @@ Add domains to the queue to be retrieved
import csv
import gzip
-from fsqueue import FSQueue, ZstdJsonSerializer
-from paths import DOMAINS_PATH, DOMAINS_QUEUE_NAME, DATA_DIR
+from indexer.fsqueue import FSQueue, ZstdJsonSerializer
+from indexer.paths import DOMAINS_PATH, DOMAINS_QUEUE_NAME, DATA_DIR
BATCH_SIZE = 250
diff --git a/extract.py b/indexer/extract.py
similarity index 100%
rename from extract.py
rename to indexer/extract.py
diff --git a/extract_local.py b/indexer/extract_local.py
similarity index 100%
rename from extract_local.py
rename to indexer/extract_local.py
diff --git a/extract_process.py b/indexer/extract_process.py
similarity index 100%
rename from extract_process.py
rename to indexer/extract_process.py
diff --git a/fsqueue.py b/indexer/fsqueue.py
similarity index 100%
rename from fsqueue.py
rename to indexer/fsqueue.py
diff --git a/hn-top-domains-filtered.py b/indexer/hn-top-domains-filtered.py
similarity index 100%
rename from hn-top-domains-filtered.py
rename to indexer/hn-top-domains-filtered.py
diff --git a/index.py b/indexer/index.py
similarity index 100%
rename from index.py
rename to indexer/index.py
diff --git a/index_glob.py b/indexer/index_glob.py
similarity index 100%
rename from index_glob.py
rename to indexer/index_glob.py
diff --git a/index_queue.py b/indexer/index_queue.py
similarity index 100%
rename from index_queue.py
rename to indexer/index_queue.py
diff --git a/indexcc.py b/indexer/indexcc.py
similarity index 100%
rename from indexcc.py
rename to indexer/indexcc.py
diff --git a/paths.py b/indexer/paths.py
similarity index 100%
rename from paths.py
rename to indexer/paths.py
diff --git a/wiki.py b/indexer/wiki.py
similarity index 91%
rename from wiki.py
rename to indexer/wiki.py
index 9f4e414..a3a66ff 100644
--- a/wiki.py
+++ b/indexer/wiki.py
@@ -7,9 +7,9 @@ from urllib.parse import quote
from spacy.lang.en import English
-from index import index_titles_urls_and_extracts
+from indexer.index import index_titles_urls_and_extracts
from tinysearchengine.indexer import TinyIndexer, NUM_PAGES, PAGE_SIZE
-from paths import WIKI_TITLES_PATH, INDEX_PATH
+from indexer.paths import WIKI_TITLES_PATH, INDEX_PATH
TEXT_TAGS = ['mediawiki', 'page', 'revision', 'text']
TITLE_START = '<title>Wikipedia: '