web search done right
getting alice cuda acceleration to actually work (hopefully)
Mikkel Denkerdda4529a2eab
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rsindex 4245611b..d9725565 100644--- a/core/src/alice/raw_model.rs+++ b/core/src/alice/raw_model.rs@@ -381,6 +381,7 @@ struct Block {pub ffn: ChannelMix,device: tch::Device,+ kind: tch::Kind,}impl Block {@@ -439,6 +440,7 @@ impl Block {att,ffn,device: tch::Device::Cpu,+ kind: tch::Kind::Float,})}@@ -447,10 +449,18 @@ impl Block {x = x.to(self.device);}+ if x.kind() != self.kind {+ x = x.to_kind(self.kind);+ }+if state.device() != self.device {state = state.to(self.device);}+ if state.kind() != self.kind {+ state = state.to_kind(self.kind);+ }+x = match &self.ln0 {Some(ln) => ln.forward_t(&x, false),None => x,@@ -470,6 +480,7 @@ impl Block {fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {self.device = device;+ self.kind = kind;if let Some(ln) = &mut self.ln0 {ln.ws = ln.ws.as_ref().map(|t| t.to_kind(kind).to(device));@@ -609,6 +620,10 @@ impl RawModel {tokens = tokens.to(self.emb.ws.device());}+ if self.emb.ws.kind() != tokens.kind() {+ tokens = tokens.to_kind(self.emb.ws.kind());+ }+let mut x = self.emb.forward_t(&tokens, false);let mut state = match state {diff --git a/core/src/lib.rs b/core/src/lib.rsindex f9ff3beb..b38b2aea 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -220,7 +220,7 @@ pub struct CrawlerConfig {}#[derive(Debug, Serialize, Deserialize, Clone)]-#[serde(tag = "type", rename_all = "snake_case")]+#[serde(tag = "type", content = "args", rename_all = "snake_case")]pub enum AcceleratorDevice {Cpu,Cuda(usize),diff --git a/scripts/download_libtorch.py b/scripts/download_libtorch.pyindex 1cbbd9a0..a8c2e47c 100644--- a/scripts/download_libtorch.py+++ b/scripts/download_libtorch.py@@ -50,7 +50,7 @@ def install_libtorch(gpu=False):else:if system == "Linux":if gpu:- url = "https://download.pytorch.org/libtorch/cu118/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcu118.zip"+ url = "https://download.pytorch.org/libtorch/cu117/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcu117.zip"else:url = "https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip"else: