Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update VaultBlobReference owner change (#2159)

Leendert de Borstad026ae29c42
diff --git a/core/client/src/database/queries/LogoQueries.ts b/core/client/src/database/queries/LogoQueries.ts
index e95f3001f..1a56e1549 100644
--- a/core/client/src/database/queries/LogoQueries.ts
+++ b/core/client/src/database/queries/LogoQueries.ts
@@ -29,7 +29,7 @@ export class LogoQueries {
LIMIT 1`;
/**
- * The kind and key of an existing logo.
+ * The kind and key of an existing logo, in whichever manifest holds it.
*/
public static readonly GET_BY_ID = `
SELECT Id, Kind, Source, Name FROM Logos
@@ -48,21 +48,4 @@ export class LogoQueries {
Name = COALESCE(excluded.Name, Logos.Name),
UpdatedAt = excluded.UpdatedAt,
IsDeleted = 0`;
-
- /**
- * Every item whose logo lives outside the item's own manifest, with the kind and key needed to bring a
- * copy in.
- */
- public static readonly FIND_ITEMS_WITH_FOREIGN_LOGO = `
- SELECT i.Id, i.ManifestId, origin.Kind, origin.Source
- FROM Items i
- INNER JOIN Logos origin ON origin.Id = i.LogoId
- LEFT JOIN Logos own ON own.Id = i.LogoId AND own.ManifestId = i.ManifestId
- WHERE i.LogoId IS NOT NULL AND own.Id IS NULL AND i.IsDeleted = 0`;
-
- /**
- * Point an item at the copy of its logo that lives in its own manifest.
- */
- public static readonly REPOINT_ITEM_LOGO = `
- UPDATE Items SET LogoId = ? WHERE Id = ? AND ManifestId = ?`;
}
diff --git a/core/client/src/database/repositories/LogoRepository.ts b/core/client/src/database/repositories/LogoRepository.ts
index a1b05412a..273f71ca0 100644
--- a/core/client/src/database/repositories/LogoRepository.ts
+++ b/core/client/src/database/repositories/LogoRepository.ts
@@ -85,26 +85,6 @@ export class LogoRepository extends BaseRepository {
return rows.length > 0 ? rows[0] : null;
}
- /**
- * Bring every item's logo into the item's own manifest, for items that have crossed a manifest boundary.
- * @param currentDateTime The current date/time string for timestamps
- * @returns The number of items repointed
- */
- public async reconcileItemLogoScopes(currentDateTime: string): Promise<number> {
- const foreign = await this.run(this.query<{ Id: string; ManifestId: string; Kind: LogoKind; Source: string }>(
- LogoQueries.FIND_ITEMS_WITH_FOREIGN_LOGO
- ));
-
- let repointed = 0;
- for (const item of foreign) {
- const logoId = await this.adoptIntoScope(item.ManifestId, item.Kind, item.Source, currentDateTime);
- if (logoId) {
- repointed += await this.run(this.execute(LogoQueries.REPOINT_ITEM_LOGO, [logoId, item.Id, item.ManifestId]));
- }
- }
- return repointed;
- }
-
/**
* Get or create the logo for a kind and key inside one manifest, refreshing its image data.
*
diff --git a/core/rust/src/vault_sync/pull.rs b/core/rust/src/vault_sync/pull.rs
index f715d07c5..d25bad904 100644
--- a/core/rust/src/vault_sync/pull.rs
+++ b/core/rust/src/vault_sync/pull.rs
@@ -232,10 +232,15 @@ async fn download_referenced_blobs(ctx: &Ctx, resolved: &[ResolvedManifest], fal
let missing: Vec<StoredBlobRef> = refs.iter().filter(|r| !cache.contains_key(&r.hash)).cloned().collect();
ctx.log(format!("[V2Pull] Blob refs: {} referenced, {} cached locally, {} to download.", refs.len(), refs.len() - missing.len(), missing.len())).await;
- let batches = http::batch_by_transfer_cost(missing, |r| base64_chars(r.size_bytes));
+ // The server stores blobs per manifest, so each batch names the manifest that owns its hashes.
+ let mut batches: Vec<(String, Vec<StoredBlobRef>)> = Vec::new();
+ for entry in resolved {
+ let owned: Vec<StoredBlobRef> = missing.iter().filter(|r| owners.get(&r.hash).is_some_and(|o| o.manifest_id == entry.manifest_id)).cloned().collect();
+ batches.extend(http::batch_by_transfer_cost(owned, |r| base64_chars(r.size_bytes)).into_iter().map(|chunk| (entry.manifest_id.clone(), chunk)));
+ }
let batch_count = batches.len();
- for (index, chunk) in batches.into_iter().enumerate() {
- let blobs: Vec<BlobDto> = http::post(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;
+ for (index, (manifest_id, chunk)) in batches.into_iter().enumerate() {
+ let blobs: Vec<BlobDto> = http::post(&ctx.host, BLOBS_DOWNLOAD_ENDPOINT, &BlobHashesRequest { manifest_id, hashes: chunk.iter().map(|r| r.hash.clone()).collect() }, true).await?;
ctx.log(format!("[V2Pull] Downloaded blob batch {}/{}: requested {}, received {}.", index + 1, batch_count, chunk.len(), blobs.len())).await;
for dto in blobs {
cache.insert(dto.hash, dto.encrypted_data_base64);
diff --git a/core/rust/src/vault_sync/push.rs b/core/rust/src/vault_sync/push.rs
index 070ae84b6..5d5b5f571 100644
--- a/core/rust/src/vault_sync/push.rs
+++ b/core/rust/src/vault_sync/push.rs
@@ -331,6 +331,7 @@ impl Candidate<'_> {
/// A plaintext blob staged for upload.
struct UploadBlobEntry {
+ manifest_id: String,
bytes: Vec<u8>,
kind: String,
vek: String,
@@ -347,6 +348,19 @@ impl UploadBlobs {
fn hashes(&self, from_personal: bool) -> Vec<String> {
self.order.iter().filter(|h| self.entries[*h].from_personal == from_personal).cloned().collect()
}
+
+ /// The given hashes grouped by the manifest that owns them, in first-seen order; the server stores blobs per manifest.
+ fn by_manifest(&self, hashes: &[String]) -> Vec<(String, Vec<String>)> {
+ let mut groups: Vec<(String, Vec<String>)> = Vec::new();
+ for hash in hashes {
+ let Some(entry) = self.entries.get(hash) else { continue };
+ match groups.iter_mut().find(|(manifest_id, _)| *manifest_id == entry.manifest_id) {
+ Some((_, group)) => group.push(hash.clone()),
+ None => groups.push((entry.manifest_id.clone(), vec![hash.clone()])),
+ }
+ }
+ groups
+ }
}
/// The freshly generated key hierarchy for one migration push.
@@ -484,7 +498,7 @@ fn collect_candidates<'a>(canonicalized: &'a CanonicalizedVault, records: &'a [M
.collect()
}
-/// The plaintext blobs the candidates carry; a blob two manifests share is staged once, under the first owner's key.
+/// The plaintext blobs the candidates carry, each owned by the manifest whose salt its hash was computed with.
fn collect_upload_blobs(candidates: &[Candidate]) -> SyncResult<UploadBlobs> {
let mut blobs = UploadBlobs { entries: HashMap::new(), order: Vec::new() };
for candidate in candidates {
@@ -493,7 +507,7 @@ fn collect_upload_blobs(candidates: &[Candidate]) -> SyncResult<UploadBlobs> {
continue;
}
blobs.order.push(hash.clone());
- blobs.entries.insert(hash.clone(), UploadBlobEntry { bytes: crate::encoding::base64_decode(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });
+ blobs.entries.insert(hash.clone(), UploadBlobEntry { manifest_id: candidate.record.manifest_id.clone(), bytes: crate::encoding::base64_decode(&blob.bytes_base64)?, kind: blob.kind.clone(), vek: candidate.vek.clone(), from_personal: candidate.record.is_personal });
}
}
Ok(blobs)
@@ -579,24 +593,25 @@ async fn upload_missing_blobs(ctx: &Ctx, blobs: &UploadBlobs, baselines: &PushBa
let unknown_to_server = |hashes: &[String]| -> Vec<String> { hashes.iter().filter(|h| !baselines.known_server_hashes.contains(*h)).cloned().collect() };
let (personal_to_upload, shared_to_upload) = if gate.migrating {
- (personal_hashes.clone(), missing_on_server(ctx, unknown_to_server(&shared_hashes)).await?)
+ (personal_hashes.clone(), missing_on_server(ctx, blobs, &unknown_to_server(&shared_hashes)).await?)
} else {
- let missing: HashSet<String> = missing_on_server(ctx, unknown_to_server(&blobs.order)).await?.into_iter().collect();
+ let missing: HashSet<String> = missing_on_server(ctx, blobs, &unknown_to_server(&blobs.order)).await?.into_iter().collect();
(personal_hashes.iter().filter(|h| missing.contains(*h)).cloned().collect(), shared_hashes.iter().filter(|h| missing.contains(*h)).cloned().collect())
};
ctx.log(format!("[V2Push] Blob diff: {} blobs, uploading {} personal + {} shared{}.", blobs.order.len(), personal_to_upload.len(), shared_to_upload.len(), if gate.migrating { " (personal manifest re-encrypted, VEK migration)" } else { "" })).await;
- let mut uploaded = upload_blobs(ctx, &blobs.entries, &personal_to_upload, gate.migrating).await?;
- uploaded.extend(upload_blobs(ctx, &blobs.entries, &shared_to_upload, false).await?);
+ let mut uploaded = upload_blobs(ctx, blobs, &personal_to_upload, gate.migrating).await?;
+ uploaded.extend(upload_blobs(ctx, blobs, &shared_to_upload, false).await?);
Ok(uploaded)
}
-/// Ask the server which of the given hashes it lacks.
-async fn missing_on_server(ctx: &Ctx, hashes: Vec<String>) -> SyncResult<Vec<String>> {
- if hashes.is_empty() {
- return Ok(Vec::new());
+/// Ask the server which of the given hashes it lacks, one request per manifest that owns them.
+async fn missing_on_server(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String]) -> SyncResult<Vec<String>> {
+ let mut missing = Vec::new();
+ for (manifest_id, hashes) in blobs.by_manifest(hashes) {
+ missing.extend(http::post::<_, MissingBlobsResponse>(&ctx.host, BLOBS_MISSING_ENDPOINT, &BlobHashesRequest { manifest_id, hashes }, false).await?.missing);
}
- Ok(http::post::<_, MissingBlobsResponse>(&ctx.host, BLOBS_MISSING_ENDPOINT, &BlobHashesRequest { hashes }, false).await?.missing)
+ Ok(missing)
}
/// `POST v2/Vault`. When the server reports blobs it lacks (stale local knowledge of its blob set), upload
@@ -612,8 +627,8 @@ async fn write_vault(ctx: &Ctx, payload: &VaultWriteRequest, blobs: &UploadBlobs
}
ctx.warn(format!("[V2Sync] Server reported {} missing blob(s); uploading and retrying once.", response.missing_blob_hashes.len())).await;
let (missing_personal, missing_shared): (Vec<String>, Vec<String>) = response.missing_blob_hashes.iter().cloned().partition(|h| blobs.entries[h].from_personal);
- uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_personal, gate.migrating).await?);
- uploaded.extend(upload_blobs(ctx, &blobs.entries, &missing_shared, false).await?);
+ uploaded.extend(upload_blobs(ctx, blobs, &missing_personal, gate.migrating).await?);
+ uploaded.extend(upload_blobs(ctx, blobs, &missing_shared, false).await?);
let response: VaultWriteResponse = http::post(&ctx.host, http::VAULT_ENDPOINT, payload, true).await?;
if !response.missing_blob_hashes.is_empty() {
return Err(SyncError::MissingBlobs(response.missing_blob_hashes));
@@ -665,21 +680,23 @@ async fn complete_account_key_migration(ctx: &mut Ctx, migration: &LegacyAccount
Ok(())
}
-/// Encrypt the given blobs (each under its own key) and upload them in size-capped batches. Returns hash to
-/// ciphertext for the local encrypted blob cache.
-async fn upload_blobs(ctx: &Ctx, entries: &HashMap<String, UploadBlobEntry>, hashes: &[String], overwrite: bool) -> SyncResult<HashMap<String, String>> {
+/// Encrypt the given blobs (each under its own key) and upload them in size-capped batches per owning manifest.
+/// Returns hash to ciphertext for the local encrypted blob cache.
+async fn upload_blobs(ctx: &Ctx, blobs: &UploadBlobs, hashes: &[String], overwrite: bool) -> SyncResult<HashMap<String, String>> {
let mut ciphertexts = HashMap::new();
- let mut dtos = Vec::new();
- for hash in hashes {
- let Some(entry) = entries.get(hash) else { continue };
- let ciphertext = crypto::symmetric_encrypt_bytes(&entry.bytes, &entry.vek)?;
- ciphertexts.insert(hash.clone(), ciphertext.clone());
- dtos.push(BlobDto { hash: hash.clone(), category: entry.kind.clone(), encrypted_data_base64: ciphertext });
- }
- for batch in http::batch_by_transfer_cost(dtos, |dto| dto.encrypted_data_base64.len()) {
- let chars: usize = batch.iter().map(|dto| dto.encrypted_data_base64.len()).sum();
- ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(chars))).await;
- http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { blobs: batch, overwrite }).await?;
+ for (manifest_id, hashes) in blobs.by_manifest(hashes) {
+ let mut dtos = Vec::new();
+ for hash in hashes {
+ let entry = &blobs.entries[&hash];
+ let ciphertext = crypto::symmetric_encrypt_bytes(&entry.bytes, &entry.vek)?;
+ ciphertexts.insert(hash.clone(), ciphertext.clone());
+ dtos.push(BlobDto { hash, category: entry.kind.clone(), encrypted_data_base64: ciphertext });
+ }
+ for batch in http::batch_by_transfer_cost(dtos, |dto| dto.encrypted_data_base64.len()) {
+ let chars: usize = batch.iter().map(|dto| dto.encrypted_data_base64.len()).sum();
+ ctx.log(format!("[V2Push] Uploading blob batch: {} blobs, {}.", batch.len(), format_kb(chars))).await;
+ http::post_no_content(&ctx.host, BLOBS_ENDPOINT, &BlobUploadRequest { manifest_id: manifest_id.clone(), blobs: batch, overwrite }).await?;
+ }
}
Ok(ciphertexts)
}
diff --git a/core/rust/src/vault_sync/tests/mod.rs b/core/rust/src/vault_sync/tests/mod.rs
index ce3ec5d01..985785aea 100644
--- a/core/rust/src/vault_sync/tests/mod.rs
+++ b/core/rust/src/vault_sync/tests/mod.rs
@@ -187,6 +187,47 @@ fn dirty_client_pushes_only_what_changed() {
assert_eq!(manifest["tables"]["Items"].as_array().unwrap().len(), 2);
}
+#[test]
+fn pushed_blobs_name_the_manifest_that_owns_them() {
+ let vek = crypto::generate_key_base64();
+ let mut host = TestHost::new(&vek);
+ let server_db = test_host::open_schema_db(&host.schema_sql);
+ insert_item(&server_db, "aaaaaaaa-0000-4000-8000-000000000001", "Server item", PERSONAL_MANIFEST_ID);
+ let (status, vault) = snapshot_of(&server_db, &vek, 7, &vault_codec::generate_manifest_salt());
+ host.state.insert(state::ENCRYPTED_ACCOUNT_KEY.to_string(), json!("wrapped"));
+ host.respond("GET", "Status", status);
+ host.respond("GET", "Vault", vault);
+ host.drive(&SyncSession::new(&request("fullSync", &vek, false, 0)).unwrap());
+
+ let now = crate::timestamp::now_vault_datetime();
+ host.local
+ .execute(
+ "INSERT INTO Logos (ManifestId, Id, Source, FileData, Kind, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, 'example.com', ?, 'favicon', ?, ?, 0)",
+ rusqlite::params![PERSONAL_MANIFEST_ID, "bbbbbbbb-0000-4000-8000-000000000001", vec![1u8, 2, 3, 4], now, now],
+ )
+ .unwrap();
+ host.local.execute("UPDATE Items SET LogoId = 'bbbbbbbb-0000-4000-8000-000000000001'", []).unwrap();
+ host.store_local_as_blob();
+ host.mutation_sequence = 1;
+ host.is_dirty = true;
+ host.respond_with(Box::new(|method, path, body| (method == "POST" && path == "Vault/blobs/missing").then(|| (200, json!({ "missing": body.map(|b| b["hashes"].clone()).unwrap_or_default() })))));
+ host.respond("POST", "Vault/blobs", json!({ "acceptedCount": 1 }));
+ host.respond("POST", "Vault", json!({ "status": 0, "manifestRevisions": [{ "manifestId": PERSONAL_MANIFEST_ID, "revision": 8 }], "bucketRevisions": [], "missingBlobHashes": [] }));
+
+ let result = host.drive(&SyncSession::new(&request("fullSync", &vek, true, 1)).unwrap());
+
+ assert_eq!(result["success"], true, "{}", result);
+ let missing_checks = host.requests_to("Vault/blobs/missing");
+ assert_eq!(missing_checks.len(), 1);
+ assert_eq!(missing_checks[0].body.as_ref().unwrap()["manifestId"], PERSONAL_MANIFEST_ID);
+ let uploads = host.requests_to("Vault/blobs");
+ assert_eq!(uploads.len(), 1);
+ let upload = uploads[0].body.as_ref().unwrap();
+ assert_eq!(upload["manifestId"], PERSONAL_MANIFEST_ID);
+ assert_eq!(upload["blobs"].as_array().unwrap().len(), 1);
+ assert_eq!(upload["blobs"][0]["hash"], missing_checks[0].body.as_ref().unwrap()["hashes"][0]);
+}
+
#[test]
fn no_op_mutation_clears_the_dirty_flag_without_a_write() {
let vek = crypto::generate_key_base64();
diff --git a/core/rust/src/vault_sync/types.rs b/core/rust/src/vault_sync/types.rs
index 1bd2a5d18..356d1dbca 100644
--- a/core/rust/src/vault_sync/types.rs
+++ b/core/rust/src/vault_sync/types.rs
@@ -719,6 +719,7 @@ pub struct VaultWriteResponse {
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BlobUploadRequest {
+ pub manifest_id: String,
pub blobs: Vec<BlobDto>,
pub overwrite: bool,
}
@@ -726,6 +727,7 @@ pub struct BlobUploadRequest {
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct BlobHashesRequest {
+ pub manifest_id: String,
pub hashes: Vec<String>,
}