Skip to content

web search done right

download libtorch from python script since we need to download from pytorch website if compiling for linux

Mikkel Denker1e95f942078c
diff --git a/.cargo/config.toml b/.cargo/config.toml
index b84f548e..b389d05c 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -2,6 +2,7 @@
rustflags = ["-C", "target-cpu=native"]
[env]
-LIBTORCH = {value = ".venv/lib/python3/site-packages/torch", force=true, relative=true}
-DYLD_LIBRARY_PATH = {value = ".venv/lib/python3/site-packages/torch/lib", force=true, relative=true}
+LIBTORCH = {value = "libtorch", force=true, relative=true}
+DYLD_LIBRARY_PATH = {value = "libtorch/lib", force=true, relative=true}
+LD_LIBRARY_PATH = {value = "libtorch/lib", force=true, relative=true}
LIBSQLITE3_FLAGS = { value = "-DSQLITE_ENABLE_MATH_FUNCTIONS", force=true }
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 0f2ab876..bc55d0f5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,5 @@ notes
*.tar.gz
*.csv
ltr_scores.json
-.venv
\ No newline at end of file
+.venv
+libtorch
diff --git a/justfile b/justfile
index 8e6785c2..6f460d30 100644
--- a/justfile
+++ b/justfile
@@ -19,6 +19,7 @@
@configure *ARGS:
just setup_python_env
+ just download_libtorch {{ARGS}}
./scripts/export_crossencoder
./scripts/export_qa_model
./scripts/export_abstractive_summary_model
@@ -30,7 +31,6 @@
@centrality webgraph output:
./scripts/build_harmonic {{webgraph}} {{output}}
-
@entity:
rm -rf data/entity
cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity
@@ -38,5 +38,7 @@
@setup_python_env:
rm -rf .venv
python3 -m venv .venv
- ln -s .venv/lib/python3* .venv/lib/python3
- .venv/bin/pip install -r scripts/requirements.txt
\ No newline at end of file
+ .venv/bin/pip install -r scripts/requirements.txt
+
+@download_libtorch *ARGS:
+ .venv/bin/python3 scripts/download_libtorch.py {{ARGS}}
\ No newline at end of file
diff --git a/scripts/download_libtorch.py b/scripts/download_libtorch.py
new file mode 100644
index 00000000..d32c190e
--- /dev/null
+++ b/scripts/download_libtorch.py
@@ -0,0 +1,76 @@
+import platform
+import subprocess
+import sys
+import os
+import requests
+import zipfile
+import shutil
+from urllib.request import urlretrieve
+import argparse
+
+
+def download_file(url, filename):
+ with open(filename, "wb") as f:
+ response = requests.get(url, stream=True)
+ total = response.headers.get("content-length")
+
+ if total is None:
+ f.write(response.content)
+ else:
+ downloaded = 0
+ total = int(total)
+ for data in response.iter_content(
+ chunk_size=max(int(total / 1000), 1024 * 1024)
+ ):
+ downloaded += len(data)
+ f.write(data)
+ done = int(50 * downloaded / total)
+ sys.stdout.write("\r[{}{}]".format("█" * done, "." * (50 - done)))
+ sys.stdout.flush()
+ sys.stdout.write("\n")
+
+
+def install_libtorch(gpu=False):
+ system = platform.system()
+
+ if os.path.exists("./libtorch"):
+ return
+
+ if system == "Darwin":
+ subprocess.check_call([sys.executable, "-m", "pip", "install", "torch"])
+ site_packages = (
+ subprocess.check_output(
+ [sys.executable, "-c", "import site; print(site.getsitepackages()[0])"]
+ )
+ .decode()
+ .strip()
+ )
+ torch_path = os.path.join(site_packages, "torch")
+ shutil.copytree(torch_path, "./libtorch", dirs_exist_ok=True)
+ else:
+ if system == "Linux":
+ if gpu:
+ url = "https://download.pytorch.org/libtorch/cu118/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcu118.zip"
+ else:
+ url = "https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip"
+ else:
+ print("Unsupported system")
+ return
+ filename = "libtorch.zip"
+ print("Downloading libtorch...")
+ download_file(url, filename)
+ print("Extracting libtorch...")
+ with zipfile.ZipFile(filename, "r") as zip_ref:
+ zip_ref.extractall("./")
+ print("Done!")
+ os.remove(filename)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Download and install libtorch.")
+ parser.add_argument(
+ "--gpu", action="store_true", help="Download libtorch with CUDA support."
+ )
+ args, _ = parser.parse_known_args()
+
+ install_libtorch(args.gpu)