web search done right
download libtorch from python script since we need to download from pytorch website if compiling for linux
Mikkel Denker1e95f942078c
diff --git a/.cargo/config.toml b/.cargo/config.tomlindex b84f548e..b389d05c 100644--- a/.cargo/config.toml+++ b/.cargo/config.toml@@ -2,6 +2,7 @@rustflags = ["-C", "target-cpu=native"][env]-LIBTORCH = {value = ".venv/lib/python3/site-packages/torch", force=true, relative=true}-DYLD_LIBRARY_PATH = {value = ".venv/lib/python3/site-packages/torch/lib", force=true, relative=true}+LIBTORCH = {value = "libtorch", force=true, relative=true}+DYLD_LIBRARY_PATH = {value = "libtorch/lib", force=true, relative=true}+LD_LIBRARY_PATH = {value = "libtorch/lib", force=true, relative=true}LIBSQLITE3_FLAGS = { value = "-DSQLITE_ENABLE_MATH_FUNCTIONS", force=true }\ No newline at end of filediff --git a/.gitignore b/.gitignoreindex 0f2ab876..bc55d0f5 100644--- a/.gitignore+++ b/.gitignore@@ -11,4 +11,5 @@ notes*.tar.gz*.csvltr_scores.json-.venv\ No newline at end of file+.venv+libtorchdiff --git a/justfile b/justfileindex 8e6785c2..6f460d30 100644--- a/justfile+++ b/justfile@@ -19,6 +19,7 @@@configure *ARGS:just setup_python_env+ just download_libtorch {{ARGS}}./scripts/export_crossencoder./scripts/export_qa_model./scripts/export_abstractive_summary_model@@ -30,7 +31,6 @@@centrality webgraph output:./scripts/build_harmonic {{webgraph}} {{output}}-@entity:rm -rf data/entitycargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity@@ -38,5 +38,7 @@@setup_python_env:rm -rf .venvpython3 -m venv .venv- ln -s .venv/lib/python3* .venv/lib/python3- .venv/bin/pip install -r scripts/requirements.txt\ No newline at end of file+ .venv/bin/pip install -r scripts/requirements.txt++@download_libtorch *ARGS:+ .venv/bin/python3 scripts/download_libtorch.py {{ARGS}}\ No newline at end of filediff --git a/scripts/download_libtorch.py b/scripts/download_libtorch.pynew file mode 100644index 00000000..d32c190e--- /dev/null+++ b/scripts/download_libtorch.py@@ -0,0 +1,76 @@+import platform+import subprocess+import sys+import os+import requests+import zipfile+import shutil+from urllib.request import urlretrieve+import argparse+++def download_file(url, filename):+ with open(filename, "wb") as f:+ response = requests.get(url, stream=True)+ total = response.headers.get("content-length")++ if total is None:+ f.write(response.content)+ else:+ downloaded = 0+ total = int(total)+ for data in response.iter_content(+ chunk_size=max(int(total / 1000), 1024 * 1024)+ ):+ downloaded += len(data)+ f.write(data)+ done = int(50 * downloaded / total)+ sys.stdout.write("\r[{}{}]".format("█" * done, "." * (50 - done)))+ sys.stdout.flush()+ sys.stdout.write("\n")+++def install_libtorch(gpu=False):+ system = platform.system()++ if os.path.exists("./libtorch"):+ return++ if system == "Darwin":+ subprocess.check_call([sys.executable, "-m", "pip", "install", "torch"])+ site_packages = (+ subprocess.check_output(+ [sys.executable, "-c", "import site; print(site.getsitepackages()[0])"]+ )+ .decode()+ .strip()+ )+ torch_path = os.path.join(site_packages, "torch")+ shutil.copytree(torch_path, "./libtorch", dirs_exist_ok=True)+ else:+ if system == "Linux":+ if gpu:+ url = "https://download.pytorch.org/libtorch/cu118/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcu118.zip"+ else:+ url = "https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip"+ else:+ print("Unsupported system")+ return+ filename = "libtorch.zip"+ print("Downloading libtorch...")+ download_file(url, filename)+ print("Extracting libtorch...")+ with zipfile.ZipFile(filename, "r") as zip_ref:+ zip_ref.extractall("./")+ print("Done!")+ os.remove(filename)+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Download and install libtorch.")+ parser.add_argument(+ "--gpu", action="store_true", help="Download libtorch with CUDA support."+ )+ args, _ = parser.parse_known_args()++ install_libtorch(args.gpu)