diff --git a/services/backup/src/database/backup_item.rs b/services/backup/src/database/backup_item.rs index acb991604..21b48b008 100644 --- a/services/backup/src/database/backup_item.rs +++ b/services/backup/src/database/backup_item.rs @@ -1,200 +1,203 @@ +use crate::constants::backup_table; use aws_sdk_dynamodb::types::AttributeValue; use chrono::{DateTime, Utc}; use comm_lib::{ blob::{client::BlobServiceClient, types::BlobInfo}, database::{AttributeTryInto, DBItemError, TryFromAttribute}, }; use std::collections::HashMap; -use crate::constants::backup_table; - #[derive(Clone, Debug)] pub struct BackupItem { pub user_id: String, pub backup_id: String, pub created: DateTime, pub user_keys: BlobInfo, pub user_data: BlobInfo, pub attachments: Vec, } impl BackupItem { pub fn new( user_id: String, backup_id: String, user_keys: BlobInfo, user_data: BlobInfo, attachments: Vec, ) -> Self { BackupItem { user_id, backup_id, created: chrono::Utc::now(), user_keys, user_data, attachments, } } - pub async fn revoke_holders(self, blob_client: &BlobServiceClient) { - blob_client - .schedule_revoke_holder(self.user_keys.blob_hash, self.user_keys.holder); + pub fn revoke_holders(&self, blob_client: &BlobServiceClient) { + blob_client.schedule_revoke_holder( + &self.user_keys.blob_hash, + &self.user_keys.holder, + ); - blob_client - .schedule_revoke_holder(self.user_data.blob_hash, self.user_data.holder); + blob_client.schedule_revoke_holder( + &self.user_data.blob_hash, + &self.user_data.holder, + ); - for attachment_info in self.attachments { + for attachment_info in &self.attachments { blob_client.schedule_revoke_holder( - attachment_info.blob_hash, - attachment_info.holder, + &attachment_info.blob_hash, + &attachment_info.holder, ); } } pub fn item_key( user_id: &str, backup_id: &str, ) -> HashMap { HashMap::from([ ( backup_table::attr::USER_ID.to_string(), AttributeValue::S(user_id.to_string()), ), ( backup_table::attr::BACKUP_ID.to_string(), AttributeValue::S(backup_id.to_string()), ), ]) } } impl From for HashMap { fn from(value: BackupItem) -> Self { let mut attrs = HashMap::from([ ( backup_table::attr::USER_ID.to_string(), AttributeValue::S(value.user_id), ), ( backup_table::attr::BACKUP_ID.to_string(), AttributeValue::S(value.backup_id), ), ( backup_table::attr::CREATED.to_string(), AttributeValue::S(value.created.to_rfc3339()), ), ( backup_table::attr::USER_KEYS.to_string(), value.user_keys.into(), ), ( backup_table::attr::USER_DATA.to_string(), value.user_data.into(), ), ]); if !value.attachments.is_empty() { attrs.insert( backup_table::attr::ATTACHMENTS.to_string(), AttributeValue::L( value .attachments .into_iter() .map(AttributeValue::from) .collect(), ), ); } attrs } } impl TryFrom> for BackupItem { type Error = DBItemError; fn try_from( mut value: HashMap, ) -> Result { let user_id = String::try_from_attr( backup_table::attr::USER_ID, value.remove(backup_table::attr::USER_ID), )?; let backup_id = String::try_from_attr( backup_table::attr::BACKUP_ID, value.remove(backup_table::attr::BACKUP_ID), )?; let created = DateTime::::try_from_attr( backup_table::attr::CREATED, value.remove(backup_table::attr::CREATED), )?; let user_keys = BlobInfo::try_from_attr( backup_table::attr::USER_KEYS, value.remove(backup_table::attr::USER_KEYS), )?; let user_data = BlobInfo::try_from_attr( backup_table::attr::USER_DATA, value.remove(backup_table::attr::USER_DATA), )?; let attachments = value.remove(backup_table::attr::ATTACHMENTS); let attachments = if attachments.is_some() { attachments.attr_try_into(backup_table::attr::ATTACHMENTS)? } else { Vec::new() }; Ok(BackupItem { user_id, backup_id, created, user_keys, user_data, attachments, }) } } /// Corresponds to the items in the [`crate::constants::BACKUP_TABLE_INDEX_USERID_CREATED`] /// global index #[derive(Clone, Debug)] pub struct OrderedBackupItem { pub user_id: String, pub created: DateTime, pub backup_id: String, pub user_keys: BlobInfo, } impl TryFrom> for OrderedBackupItem { type Error = DBItemError; fn try_from( mut value: HashMap, ) -> Result { let user_id = String::try_from_attr( backup_table::attr::USER_ID, value.remove(backup_table::attr::USER_ID), )?; let created = DateTime::::try_from_attr( backup_table::attr::CREATED, value.remove(backup_table::attr::CREATED), )?; let backup_id = String::try_from_attr( backup_table::attr::BACKUP_ID, value.remove(backup_table::attr::BACKUP_ID), )?; let user_keys = BlobInfo::try_from_attr( backup_table::attr::USER_KEYS, value.remove(backup_table::attr::USER_KEYS), )?; Ok(OrderedBackupItem { user_id, created, backup_id, user_keys, }) } } diff --git a/services/backup/src/database/log_item.rs b/services/backup/src/database/log_item.rs index 196636ac9..d4497d65e 100644 --- a/services/backup/src/database/log_item.rs +++ b/services/backup/src/database/log_item.rs @@ -1,142 +1,156 @@ use crate::constants::{log_table::attr, LOG_BACKUP_ID_SEPARATOR}; use aws_sdk_dynamodb::types::AttributeValue; use comm_lib::{ blob::{ client::{BlobServiceClient, BlobServiceError}, types::BlobInfo, }, constants::DDB_ITEM_SIZE_LIMIT, database::{ blob::BlobOrDBContent, calculate_size_in_db, parse_int_attribute, AttributeExtractor, AttributeTryInto, DBItemAttributeError, DBItemError, Value, }, }; use std::collections::HashMap; use tracing::debug; #[derive(Clone, Debug)] pub struct LogItem { pub user_id: String, pub backup_id: String, pub log_id: usize, pub content: BlobOrDBContent, pub attachments: Vec, } impl LogItem { pub async fn ensure_size_constraints( &mut self, blob_client: &BlobServiceClient, ) -> Result<(), BlobServiceError> { if let Ok(size) = calculate_size_in_db(&self.clone().into()) { if size < DDB_ITEM_SIZE_LIMIT { return Ok(()); }; } debug!( log_id = ?self.log_id, "Log content exceeds DDB item size limit, moving to blob storage" ); self.content.move_to_blob(blob_client).await } pub fn partition_key( user_id: impl Into, backup_id: impl Into, ) -> String { format!( "{}{}{}", user_id.into(), LOG_BACKUP_ID_SEPARATOR, backup_id.into(), ) } + pub fn revoke_holders(&self, blob_client: &BlobServiceClient) { + if let BlobOrDBContent::Blob(content_info) = &self.content { + blob_client + .schedule_revoke_holder(&content_info.blob_hash, &content_info.holder); + } + + for attachment_info in &self.attachments { + blob_client.schedule_revoke_holder( + &attachment_info.blob_hash, + &attachment_info.holder, + ); + } + } + pub fn item_key( user_id: impl Into, backup_id: impl Into, log_id: usize, ) -> HashMap { HashMap::from([ ( attr::BACKUP_ID.to_string(), AttributeValue::S(Self::partition_key(user_id, backup_id)), ), ( attr::LOG_ID.to_string(), AttributeValue::N(log_id.to_string()), ), ]) } } impl From for HashMap { fn from(value: LogItem) -> Self { let mut attrs = LogItem::item_key(value.user_id, value.backup_id, value.log_id); let (content_attr_name, content_attr) = value .content .into_attr_pair(attr::CONTENT_BLOB_INFO, attr::CONTENT_DB); attrs.insert(content_attr_name, content_attr); if !value.attachments.is_empty() { attrs.insert( attr::ATTACHMENTS.to_string(), AttributeValue::L( value .attachments .into_iter() .map(AttributeValue::from) .collect(), ), ); } attrs } } impl TryFrom> for LogItem { type Error = DBItemError; fn try_from( mut value: HashMap, ) -> Result { let id: String = value.take_attr(attr::BACKUP_ID)?; let (user_id, backup_id) = match &id.split(LOG_BACKUP_ID_SEPARATOR).collect::>()[..] { &[user_id, backup_id] => (user_id.to_string(), backup_id.to_string()), _ => { return Err(DBItemError::new( attr::BACKUP_ID.to_string(), Value::String(id), DBItemAttributeError::InvalidValue, )) } }; let log_id = parse_int_attribute(attr::LOG_ID, value.remove(attr::LOG_ID))?; let content = BlobOrDBContent::parse_from_attrs( &mut value, attr::CONTENT_BLOB_INFO, attr::CONTENT_DB, )?; let attachments = value.remove(attr::ATTACHMENTS); let attachments = if attachments.is_some() { attachments.attr_try_into(attr::ATTACHMENTS)? } else { Vec::new() }; Ok(LogItem { user_id, backup_id, log_id, content, attachments, }) } } diff --git a/services/backup/src/database/mod.rs b/services/backup/src/database/mod.rs index 3de1dd7b3..7c2c1e1f7 100644 --- a/services/backup/src/database/mod.rs +++ b/services/backup/src/database/mod.rs @@ -1,325 +1,343 @@ pub mod backup_item; pub mod log_item; use self::{ backup_item::{BackupItem, OrderedBackupItem}, log_item::LogItem, }; use crate::constants::{backup_table, log_table, LOG_DEFAULT_PAGE_SIZE}; use aws_sdk_dynamodb::{ operation::get_item::GetItemOutput, types::{AttributeValue, DeleteRequest, ReturnValue, WriteRequest}, }; -use comm_lib::database::{ - self, batch_operations::ExponentialBackoffConfig, parse_int_attribute, Error, +use comm_lib::{ + blob::client::BlobServiceClient, + database::{ + self, batch_operations::ExponentialBackoffConfig, parse_int_attribute, + Error, + }, }; use tracing::{error, trace, warn}; #[derive(Clone)] pub struct DatabaseClient { client: aws_sdk_dynamodb::Client, } impl DatabaseClient { pub fn new(aws_config: &aws_types::SdkConfig) -> Self { DatabaseClient { client: aws_sdk_dynamodb::Client::new(aws_config), } } } /// Backup functions impl DatabaseClient { pub async fn put_backup_item( &self, backup_item: BackupItem, ) -> Result<(), Error> { let item = backup_item.into(); self .client .put_item() .table_name(backup_table::TABLE_NAME) .set_item(Some(item)) .send() .await .map_err(|e| { error!("DynamoDB client failed to put backup item"); Error::AwsSdk(e.into()) })?; Ok(()) } pub async fn find_backup_item( &self, user_id: &str, backup_id: &str, ) -> Result, Error> { let item_key = BackupItem::item_key(user_id, backup_id); let output = self .client .get_item() .table_name(backup_table::TABLE_NAME) .set_key(Some(item_key)) .send() .await .map_err(|e| { error!("DynamoDB client failed to find backup item"); Error::AwsSdk(e.into()) })?; let GetItemOutput { item: Some(item), .. } = output else { return Ok(None); }; let backup_item = item.try_into()?; Ok(Some(backup_item)) } pub async fn find_last_backup_item( &self, user_id: &str, ) -> Result, Error> { let response = self .client .query() .table_name(backup_table::TABLE_NAME) .index_name(backup_table::CREATED_INDEX) .key_condition_expression("#userID = :valueToMatch") .expression_attribute_names("#userID", backup_table::attr::USER_ID) .expression_attribute_values( ":valueToMatch", AttributeValue::S(user_id.to_string()), ) .limit(1) .scan_index_forward(false) .send() .await .map_err(|e| { error!("DynamoDB client failed to find last backup"); Error::AwsSdk(e.into()) })?; match response.items.unwrap_or_default().pop() { Some(item) => { let backup_item = item.try_into()?; Ok(Some(backup_item)) } None => Ok(None), } } pub async fn remove_backup_item( &self, user_id: &str, backup_id: &str, + blob_client: &BlobServiceClient, ) -> Result, Error> { let item_key = BackupItem::item_key(user_id, backup_id); let response = self .client .delete_item() .table_name(backup_table::TABLE_NAME) .set_key(Some(item_key)) .return_values(ReturnValue::AllOld) .send() .await .map_err(|e| { error!("DynamoDB client failed to remove backup item"); Error::AwsSdk(e.into()) })?; let result = response .attributes .map(BackupItem::try_from) .transpose() .map_err(Error::from)?; - self.remove_log_items_for_backup(user_id, backup_id).await?; + if let Some(backup_item) = &result { + backup_item.revoke_holders(blob_client); + } + + self + .remove_log_items_for_backup(user_id, backup_id, blob_client) + .await?; Ok(result) } /// For the purposes of the initial backup version this function /// removes all backups except for the latest one pub async fn remove_old_backups( &self, user_id: &str, + blob_client: &BlobServiceClient, ) -> Result, Error> { let response = self .client .query() .table_name(backup_table::TABLE_NAME) .index_name(backup_table::CREATED_INDEX) .key_condition_expression("#userID = :valueToMatch") .expression_attribute_names("#userID", backup_table::attr::USER_ID) .expression_attribute_values( ":valueToMatch", AttributeValue::S(user_id.to_string()), ) .scan_index_forward(false) .send() .await .map_err(|e| { error!("DynamoDB client failed to fetch backups"); Error::AwsSdk(e.into()) })?; if response.last_evaluated_key().is_some() { // In the intial version of the backup service this function will be run // for every new backup (each user only has one backup), so this shouldn't // happen warn!("Not all old backups have been cleaned up"); } let items = response .items .unwrap_or_default() .into_iter() .map(OrderedBackupItem::try_from) .collect::, _>>()?; let mut removed_backups = vec![]; let Some(latest) = items.iter().map(|item| item.created).max() else { return Ok(removed_backups); }; for item in items { if item.created == latest { trace!( "Skipping removal of the latest backup item: {}", item.backup_id ); continue; } trace!("Removing backup item: {item:?}"); - if let Some(backup) = - self.remove_backup_item(user_id, &item.backup_id).await? + if let Some(backup) = self + .remove_backup_item(user_id, &item.backup_id, blob_client) + .await? { removed_backups.push(backup); } else { warn!("Backup was found during query, but wasn't found when deleting") }; } Ok(removed_backups) } } /// Backup log functions impl DatabaseClient { pub async fn put_log_item(&self, log_item: LogItem) -> Result<(), Error> { let item = log_item.into(); self .client .put_item() .table_name(log_table::TABLE_NAME) .set_item(Some(item)) .send() .await .map_err(|e| { error!("DynamoDB client failed to put log item"); Error::AwsSdk(e.into()) })?; Ok(()) } pub async fn fetch_log_items( &self, user_id: &str, backup_id: &str, from_id: Option, ) -> Result<(Vec, Option), Error> { let id = LogItem::partition_key(user_id, backup_id); let mut query = self .client .query() .table_name(log_table::TABLE_NAME) .key_condition_expression("#backupID = :valueToMatch") .expression_attribute_names("#backupID", log_table::attr::BACKUP_ID) .expression_attribute_values( ":valueToMatch", AttributeValue::S(id.clone()), ) .limit(LOG_DEFAULT_PAGE_SIZE); if let Some(from_id) = from_id { query = query .exclusive_start_key(log_table::attr::BACKUP_ID, AttributeValue::S(id)) .exclusive_start_key( log_table::attr::LOG_ID, AttributeValue::N(from_id.to_string()), ); } let response = query.send().await.map_err(|e| { error!("DynamoDB client failed to fetch logs"); Error::AwsSdk(e.into()) })?; let last_id = response .last_evaluated_key() .map(|key| { parse_int_attribute( log_table::attr::LOG_ID, key.get(log_table::attr::LOG_ID).cloned(), ) }) .transpose()?; let items = response .items .unwrap_or_default() .into_iter() .map(LogItem::try_from) .collect::, _>>()?; Ok((items, last_id)) } pub async fn remove_log_items_for_backup( &self, user_id: &str, backup_id: &str, + blob_client: &BlobServiceClient, ) -> Result<(), Error> { let (mut items, mut last_id) = self.fetch_log_items(user_id, backup_id, None).await?; while last_id.is_some() { let (mut new_items, new_last_id) = self.fetch_log_items(user_id, backup_id, last_id).await?; items.append(&mut new_items); last_id = new_last_id; } + for log_item in &items { + log_item.revoke_holders(blob_client); + } + let write_requests = items .into_iter() .map(|key| { DeleteRequest::builder() .set_key(Some(LogItem::item_key(user_id, key.backup_id, key.log_id))) .build() }) .map(|request| WriteRequest::builder().delete_request(request).build()) .collect::>(); database::batch_operations::batch_write( &self.client, log_table::TABLE_NAME, write_requests, ExponentialBackoffConfig::default(), ) .await?; Ok(()) } } diff --git a/services/backup/src/http/handlers/backup.rs b/services/backup/src/http/handlers/backup.rs index bb8f76349..45eea5e29 100644 --- a/services/backup/src/http/handlers/backup.rs +++ b/services/backup/src/http/handlers/backup.rs @@ -1,317 +1,314 @@ use actix_web::{ error::ErrorBadRequest, web::{self, Bytes}, HttpResponse, Responder, }; use comm_lib::{ auth::UserIdentity, backup::LatestBackupIDResponse, blob::{client::BlobServiceClient, types::BlobInfo}, http::multipart::{get_named_text_field, get_text_field}, tools::Defer, }; use std::convert::Infallible; use tokio_stream::{wrappers::ReceiverStream, StreamExt}; use tracing::{info, instrument, trace, warn}; use crate::{ database::{backup_item::BackupItem, DatabaseClient}, error::BackupError, }; #[instrument(skip_all, fields(backup_id))] pub async fn upload( user: UserIdentity, blob_client: web::Data, db_client: web::Data, mut multipart: actix_multipart::Multipart, ) -> actix_web::Result { let backup_id = get_named_text_field("backup_id", &mut multipart).await?; tracing::Span::current().record("backup_id", &backup_id); info!("Backup data upload started"); let (user_keys_blob_info, user_keys_revoke) = forward_field_to_blob( &mut multipart, &blob_client, "user_keys_hash", "user_keys", ) .await?; let (user_data_blob_info, user_data_revoke) = forward_field_to_blob( &mut multipart, &blob_client, "user_data_hash", "user_data", ) .await?; let attachments_hashes: Vec = match get_text_field(&mut multipart).await? { Some((name, attachments)) => { if name != "attachments" { warn!( name, "Malformed request: 'attachments' text field expected." ); return Err(ErrorBadRequest("Bad request")); } attachments.lines().map(ToString::to_string).collect() } None => Vec::new(), }; let mut attachments = Vec::new(); let mut attachments_revokes = Vec::new(); for attachment_hash in attachments_hashes { let (holder, revoke) = create_attachment_holder(&attachment_hash, &blob_client).await?; attachments.push(BlobInfo { blob_hash: attachment_hash, holder, }); attachments_revokes.push(revoke); } let item = BackupItem::new( user.user_id.clone(), backup_id, user_keys_blob_info, user_data_blob_info, attachments, ); db_client .put_backup_item(item) .await .map_err(BackupError::from)?; user_keys_revoke.cancel(); user_data_revoke.cancel(); for attachment_revoke in attachments_revokes { attachment_revoke.cancel(); } - for backup in db_client - .remove_old_backups(&user.user_id) + db_client + .remove_old_backups(&user.user_id, &blob_client) .await - .map_err(BackupError::from)? - { - backup.revoke_holders(&blob_client).await; - } + .map_err(BackupError::from)?; Ok(HttpResponse::Ok().finish()) } #[instrument(skip_all, fields(hash_field_name, data_field_name))] async fn forward_field_to_blob<'revoke, 'blob: 'revoke>( multipart: &mut actix_multipart::Multipart, blob_client: &'blob web::Data, hash_field_name: &str, data_field_name: &str, ) -> actix_web::Result<(BlobInfo, Defer<'revoke>)> { trace!("Reading blob fields: {hash_field_name:?}, {data_field_name:?}"); let blob_hash = get_named_text_field(hash_field_name, multipart).await?; let Some(mut field) = multipart.try_next().await? else { warn!("Malformed request: expected a field."); return Err(ErrorBadRequest("Bad request"))?; }; if field.name() != data_field_name { warn!( hash_field_name, "Malformed request: '{data_field_name}' data field expected." ); return Err(ErrorBadRequest("Bad request"))?; } let blob_info = BlobInfo { blob_hash, holder: uuid::Uuid::new_v4().to_string(), }; // [`actix_multipart::Multipart`] isn't [`std::marker::Send`], and so we cannot pass it to the blob client directly. // Instead we have to forward it to a channel and create stream from the receiver. let (tx, rx) = tokio::sync::mpsc::channel(1); let receive_promise = async move { trace!("Receiving blob data"); // [`actix_multipart::MultipartError`] isn't [`std::marker::Send`] so we return it here, and pass [`Infallible`] // as the error to the channel while let Some(chunk) = field.try_next().await? { if let Err(err) = tx.send(Result::::Ok(chunk)).await { warn!("Error when sending data through a channel: '{err}'"); // Error here means that the channel has been closed from the blob client side. We don't want to return an error // here, because `tokio::try_join!` only returns the first error it receives and we want to prioritize the backup // client error. break; } } trace!("Finished receiving blob data"); Result::<(), actix_web::Error>::Ok(()) }; let data_stream = ReceiverStream::new(rx); let send_promise = async { blob_client .simple_put(&blob_info.blob_hash, &blob_info.holder, data_stream) .await .map_err(BackupError::from)?; Ok(()) }; tokio::try_join!(receive_promise, send_promise)?; let revoke_info = blob_info.clone(); let revoke_holder = Defer::new(|| { blob_client .schedule_revoke_holder(revoke_info.blob_hash, revoke_info.holder) }); Ok((blob_info, revoke_holder)) } #[instrument(skip_all)] async fn create_attachment_holder<'revoke, 'blob: 'revoke>( attachment: &str, blob_client: &'blob web::Data, ) -> Result<(String, Defer<'revoke>), BackupError> { let holder = uuid::Uuid::new_v4().to_string(); if !blob_client .assign_holder(attachment, &holder) .await .map_err(BackupError::from)? { warn!("Blob attachment with hash {attachment:?} doesn't exist"); } let revoke_hash = attachment.to_string(); let revoke_holder = holder.clone(); let revoke_holder = Defer::new(|| { blob_client.schedule_revoke_holder(revoke_hash, revoke_holder) }); Ok((holder, revoke_holder)) } #[instrument(skip_all, fields(backup_id = %path))] pub async fn download_user_keys( user: UserIdentity, path: web::Path, blob_client: web::Data, db_client: web::Data, ) -> actix_web::Result { info!("Download user keys request"); let backup_id = path.into_inner(); download_user_blob( |item| &item.user_keys, &user.user_id, &backup_id, blob_client, db_client, ) .await } #[instrument(skip_all, fields(backup_id = %path))] pub async fn download_user_data( user: UserIdentity, path: web::Path, blob_client: web::Data, db_client: web::Data, ) -> actix_web::Result { info!("Download user data request"); let backup_id = path.into_inner(); download_user_blob( |item| &item.user_data, &user.user_id, &backup_id, blob_client, db_client, ) .await } pub async fn download_user_blob( data_extractor: impl FnOnce(&BackupItem) -> &BlobInfo, user_id: &str, backup_id: &str, blob_client: web::Data, db_client: web::Data, ) -> actix_web::Result { let backup_item = db_client .find_backup_item(user_id, backup_id) .await .map_err(BackupError::from)? .ok_or(BackupError::NoBackup)?; let stream = blob_client .get(&data_extractor(&backup_item).blob_hash) .await .map_err(BackupError::from)?; Ok( HttpResponse::Ok() .content_type("application/octet-stream") .streaming(stream), ) } #[instrument(skip_all, fields(username = %path))] pub async fn get_latest_backup_id( path: web::Path, db_client: web::Data, ) -> actix_web::Result { let username = path.into_inner(); // Treat username as user_id in the initial version let user_id = username; let Some(backup_item) = db_client .find_last_backup_item(&user_id) .await .map_err(BackupError::from)? else { return Err(BackupError::NoBackup.into()); }; let response = LatestBackupIDResponse { backup_id: backup_item.backup_id, }; Ok(web::Json(response)) } #[instrument(skip_all, fields(username = %path))] pub async fn download_latest_backup_keys( path: web::Path, db_client: web::Data, blob_client: web::Data, ) -> actix_web::Result { let username = path.into_inner(); // Treat username as user_id in the initial version let user_id = username; let Some(backup_item) = db_client .find_last_backup_item(&user_id) .await .map_err(BackupError::from)? else { return Err(BackupError::NoBackup.into()); }; let stream = blob_client .get(&backup_item.user_keys.blob_hash) .await .map_err(BackupError::from)?; Ok( HttpResponse::Ok() .content_type("application/octet-stream") .streaming(stream), ) }