Skip to content

web search done right

getting alice cuda acceleration to actually work (hopefully)

Mikkel Denkerdda4529a2eab
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rs
index 4245611b..d9725565 100644
--- a/core/src/alice/raw_model.rs
+++ b/core/src/alice/raw_model.rs
@@ -381,6 +381,7 @@ struct Block {
pub ffn: ChannelMix,
device: tch::Device,
+ kind: tch::Kind,
}
impl Block {
@@ -439,6 +440,7 @@ impl Block {
att,
ffn,
device: tch::Device::Cpu,
+ kind: tch::Kind::Float,
})
}
@@ -447,10 +449,18 @@ impl Block {
x = x.to(self.device);
}
+ if x.kind() != self.kind {
+ x = x.to_kind(self.kind);
+ }
+
if state.device() != self.device {
state = state.to(self.device);
}
+ if state.kind() != self.kind {
+ state = state.to_kind(self.kind);
+ }
+
x = match &self.ln0 {
Some(ln) => ln.forward_t(&x, false),
None => x,
@@ -470,6 +480,7 @@ impl Block {
fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
self.device = device;
+ self.kind = kind;
if let Some(ln) = &mut self.ln0 {
ln.ws = ln.ws.as_ref().map(|t| t.to_kind(kind).to(device));
@@ -609,6 +620,10 @@ impl RawModel {
tokens = tokens.to(self.emb.ws.device());
}
+ if self.emb.ws.kind() != tokens.kind() {
+ tokens = tokens.to_kind(self.emb.ws.kind());
+ }
+
let mut x = self.emb.forward_t(&tokens, false);
let mut state = match state {
diff --git a/core/src/lib.rs b/core/src/lib.rs
index f9ff3beb..b38b2aea 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -220,7 +220,7 @@ pub struct CrawlerConfig {
}
#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(tag = "type", rename_all = "snake_case")]
+#[serde(tag = "type", content = "args", rename_all = "snake_case")]
pub enum AcceleratorDevice {
Cpu,
Cuda(usize),
diff --git a/scripts/download_libtorch.py b/scripts/download_libtorch.py
index 1cbbd9a0..a8c2e47c 100644
--- a/scripts/download_libtorch.py
+++ b/scripts/download_libtorch.py
@@ -50,7 +50,7 @@ def install_libtorch(gpu=False):
else:
if system == "Linux":
if gpu:
- url = "https://download.pytorch.org/libtorch/cu118/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcu118.zip"
+ url = "https://download.pytorch.org/libtorch/cu117/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcu117.zip"
else:
url = "https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip"
else: