diff --git a/services/backup/src/constants.rs b/services/backup/src/constants.rs --- a/services/backup/src/constants.rs +++ b/services/backup/src/constants.rs @@ -27,10 +27,14 @@ } } -pub const LOG_TABLE_NAME: &str = "backup-service-log"; -pub const LOG_TABLE_FIELD_BACKUP_ID: &str = "backupID"; -pub const LOG_TABLE_FIELD_LOG_ID: &str = "logID"; -pub const LOG_TABLE_FIELD_PERSISTED_IN_BLOB: &str = "persistedInBlob"; -pub const LOG_TABLE_FIELD_VALUE: &str = "value"; -pub const LOG_TABLE_FIELD_ATTACHMENT_HOLDERS: &str = "attachmentHolders"; -pub const LOG_TABLE_FIELD_DATA_HASH: &str = "dataHash"; +pub mod log_table { + pub const TABLE_NAME: &str = "backup-service-log"; + + pub mod attr { + pub const BACKUP_ID: &str = "backupID"; + pub const LOG_ID: &str = "logID"; + pub const CONTENT_DB: &str = "content"; + pub const CONTENT_BLOB_INFO: &str = "blobInfo"; + pub const ATTACHMENTS: &str = "attachments"; + } +} diff --git a/services/backup/src/database/log_item.rs b/services/backup/src/database/log_item.rs --- a/services/backup/src/database/log_item.rs +++ b/services/backup/src/database/log_item.rs @@ -1,105 +1,107 @@ -use std::collections::HashMap; - +use crate::constants::log_table::attr; use aws_sdk_dynamodb::types::AttributeValue; -use comm_lib::database::{DBItemError, TryFromAttribute}; - -use crate::constants::{ - LOG_TABLE_FIELD_ATTACHMENT_HOLDERS, LOG_TABLE_FIELD_BACKUP_ID, - LOG_TABLE_FIELD_DATA_HASH, LOG_TABLE_FIELD_LOG_ID, - LOG_TABLE_FIELD_PERSISTED_IN_BLOB, LOG_TABLE_FIELD_VALUE, +use comm_lib::{ + blob::{ + client::{BlobServiceClient, BlobServiceError}, + types::BlobInfo, + }, + constants::DDB_ITEM_SIZE_LIMIT, + database::{ + blob::BlobOrDBContent, calculate_size_in_db, parse_int_attribute, + AttributeExtractor, AttributeTryInto, DBItemError, + }, }; +use std::collections::HashMap; +use tracing::debug; #[derive(Clone, Debug)] pub struct LogItem { pub backup_id: String, - pub log_id: String, - pub persisted_in_blob: bool, - pub value: String, - pub attachment_holders: String, - pub data_hash: String, + pub log_id: usize, + pub content: BlobOrDBContent, + pub attachments: Vec, } impl LogItem { - /// Calculates size based on raw log item components, - /// without allocating a new item - pub fn size_from_components( - backup_id: &str, - log_id: &str, - log_hash: &str, - data: &[u8], - ) -> usize { - let mut size: usize = LOG_ITEM_HEADERS_SIZE; - size += backup_id.as_bytes().len(); - size += log_id.as_bytes().len(); - size += data.len(); - size += log_hash.as_bytes().len(); + pub async fn ensure_size_constraints( + &mut self, + blob_client: &BlobServiceClient, + ) -> Result<(), BlobServiceError> { + if let Ok(size) = calculate_size_in_db(&self.clone().into()) { + if size < DDB_ITEM_SIZE_LIMIT { + return Ok(()); + }; + } - // persistent in blob, attachment holders, use defaults here - size += false.to_string().as_bytes().len(); - size += "".as_bytes().len(); - - size + debug!( + log_id = ?self.log_id, + "Log content exceeds DDB item size limit, moving to blob storage" + ); + self.content.move_to_blob(blob_client).await } +} + +impl From for HashMap { + fn from(value: LogItem) -> Self { + let mut attrs = HashMap::from([ + ( + attr::BACKUP_ID.to_string(), + AttributeValue::S(value.backup_id), + ), + ( + attr::LOG_ID.to_string(), + AttributeValue::N(value.log_id.to_string()), + ), + ]); - /// Total size of this item in the DynamoDB table. This value must be - /// smaller than [`comm_lib::constants::DDB_ITEM_SIZE_LIMIT`] - /// in order to successfully put this item into a DynamoDB database. - pub fn total_size(&self) -> usize { - let mut size: usize = LOG_ITEM_HEADERS_SIZE; - size += self.backup_id.as_bytes().len(); - size += self.log_id.as_bytes().len(); - size += self.persisted_in_blob.to_string().as_bytes().len(); - size += self.value.as_bytes().len(); - size += self.attachment_holders.as_bytes().len(); - size += self.data_hash.as_bytes().len(); - size + let (content_attr_name, content_attr) = value + .content + .into_attr_pair(attr::CONTENT_BLOB_INFO, attr::CONTENT_DB); + attrs.insert(content_attr_name, content_attr); + + if !value.attachments.is_empty() { + attrs.insert( + attr::ATTACHMENTS.to_string(), + AttributeValue::L( + value + .attachments + .into_iter() + .map(AttributeValue::from) + .collect(), + ), + ); + } + + attrs } } -static LOG_ITEM_HEADERS_SIZE: usize = { - let mut size: usize = 0; - size += LOG_TABLE_FIELD_BACKUP_ID.as_bytes().len(); - size += LOG_TABLE_FIELD_LOG_ID.as_bytes().len(); - size += LOG_TABLE_FIELD_PERSISTED_IN_BLOB.as_bytes().len(); - size += LOG_TABLE_FIELD_VALUE.as_bytes().len(); - size += LOG_TABLE_FIELD_ATTACHMENT_HOLDERS.as_bytes().len(); - size += LOG_TABLE_FIELD_DATA_HASH.as_bytes().len(); - size -}; +impl TryFrom> for LogItem { + type Error = DBItemError; -pub fn parse_log_item( - mut item: HashMap, -) -> Result { - let backup_id = String::try_from_attr( - LOG_TABLE_FIELD_BACKUP_ID, - item.remove(LOG_TABLE_FIELD_BACKUP_ID), - )?; - let log_id = String::try_from_attr( - LOG_TABLE_FIELD_LOG_ID, - item.remove(LOG_TABLE_FIELD_LOG_ID), - )?; - let persisted_in_blob = bool::try_from_attr( - LOG_TABLE_FIELD_PERSISTED_IN_BLOB, - item.remove(LOG_TABLE_FIELD_PERSISTED_IN_BLOB), - )?; - let value = String::try_from_attr( - LOG_TABLE_FIELD_VALUE, - item.remove(LOG_TABLE_FIELD_VALUE), - )?; - let data_hash = String::try_from_attr( - LOG_TABLE_FIELD_DATA_HASH, - item.remove(LOG_TABLE_FIELD_DATA_HASH), - )?; - let attachment_holders = String::try_from_attr( - LOG_TABLE_FIELD_ATTACHMENT_HOLDERS, - item.remove(LOG_TABLE_FIELD_ATTACHMENT_HOLDERS), - )?; - Ok(LogItem { - log_id, - backup_id, - persisted_in_blob, - value, - data_hash, - attachment_holders, - }) + fn try_from( + mut value: HashMap, + ) -> Result { + let backup_id = value.take_attr(attr::BACKUP_ID)?; + let log_id = parse_int_attribute(attr::LOG_ID, value.remove(attr::LOG_ID))?; + let content = BlobOrDBContent::parse_from_attrs( + &mut value, + attr::CONTENT_BLOB_INFO, + attr::CONTENT_DB, + )?; + + let attachments = value.remove(attr::ATTACHMENTS); + let attachments = if attachments.is_some() { + attachments.attr_try_into(attr::ATTACHMENTS)? + } else { + Vec::new() + }; + + Ok(LogItem { + backup_id, + log_id, + content, + attachments, + }) + } } diff --git a/services/backup/src/database/mod.rs b/services/backup/src/database/mod.rs --- a/services/backup/src/database/mod.rs +++ b/services/backup/src/database/mod.rs @@ -1,26 +1,16 @@ pub mod backup_item; pub mod log_item; -use std::collections::HashMap; - +use self::backup_item::{BackupItem, OrderedBackupItem}; +use crate::constants::backup_table; use aws_sdk_dynamodb::{ operation::get_item::GetItemOutput, types::{AttributeValue, ReturnValue}, }; use comm_lib::database::Error; +use std::collections::HashMap; use tracing::{error, trace, warn}; -use crate::constants::{ - backup_table, LOG_TABLE_FIELD_ATTACHMENT_HOLDERS, LOG_TABLE_FIELD_BACKUP_ID, - LOG_TABLE_FIELD_DATA_HASH, LOG_TABLE_FIELD_LOG_ID, - LOG_TABLE_FIELD_PERSISTED_IN_BLOB, LOG_TABLE_FIELD_VALUE, LOG_TABLE_NAME, -}; - -use self::{ - backup_item::{BackupItem, OrderedBackupItem}, - log_item::{parse_log_item, LogItem}, -}; - #[derive(Clone)] pub struct DatabaseClient { client: aws_sdk_dynamodb::Client, @@ -227,138 +217,4 @@ ), ]) } - - // log item - pub async fn put_log_item(&self, log_item: LogItem) -> Result<(), Error> { - let item = HashMap::from([ - ( - LOG_TABLE_FIELD_BACKUP_ID.to_string(), - AttributeValue::S(log_item.backup_id), - ), - ( - LOG_TABLE_FIELD_LOG_ID.to_string(), - AttributeValue::S(log_item.log_id), - ), - ( - LOG_TABLE_FIELD_PERSISTED_IN_BLOB.to_string(), - AttributeValue::Bool(log_item.persisted_in_blob), - ), - ( - LOG_TABLE_FIELD_VALUE.to_string(), - AttributeValue::S(log_item.value), - ), - ( - LOG_TABLE_FIELD_DATA_HASH.to_string(), - AttributeValue::S(log_item.data_hash), - ), - ( - LOG_TABLE_FIELD_ATTACHMENT_HOLDERS.to_string(), - AttributeValue::S(log_item.attachment_holders), - ), - ]); - - self - .client - .put_item() - .table_name(LOG_TABLE_NAME) - .set_item(Some(item)) - .send() - .await - .map_err(|e| { - error!("DynamoDB client failed to put log item"); - Error::AwsSdk(e.into()) - })?; - - Ok(()) - } - - pub async fn find_log_item( - &self, - backup_id: &str, - log_id: &str, - ) -> Result, Error> { - let item_key = HashMap::from([ - ( - LOG_TABLE_FIELD_BACKUP_ID.to_string(), - AttributeValue::S(backup_id.to_string()), - ), - ( - LOG_TABLE_FIELD_LOG_ID.to_string(), - AttributeValue::S(log_id.to_string()), - ), - ]); - - match self - .client - .get_item() - .table_name(LOG_TABLE_NAME) - .set_key(Some(item_key)) - .send() - .await - .map_err(|e| { - error!("DynamoDB client failed to find log item"); - Error::AwsSdk(e.into()) - })? { - GetItemOutput { - item: Some(item), .. - } => { - let log_item = parse_log_item(item)?; - Ok(Some(log_item)) - } - _ => Ok(None), - } - } - - pub async fn find_log_items_for_backup( - &self, - backup_id: &str, - ) -> Result, Error> { - let response = self - .client - .query() - .table_name(LOG_TABLE_NAME) - .key_condition_expression("#backupID = :valueToMatch") - .expression_attribute_names("#backupID", LOG_TABLE_FIELD_BACKUP_ID) - .expression_attribute_values( - ":valueToMatch", - AttributeValue::S(backup_id.to_string()), - ) - .send() - .await - .map_err(|e| { - error!("DynamoDB client failed to find log items for backup"); - Error::AwsSdk(e.into()) - })?; - - if response.count == 0 { - return Ok(Vec::new()); - } - - let mut results: Vec = - Vec::with_capacity(response.count() as usize); - for item in response.items.unwrap_or_default() { - let log_item = parse_log_item(item)?; - results.push(log_item); - } - Ok(results) - } - - pub async fn remove_log_item(&self, log_id: &str) -> Result<(), Error> { - self - .client - .delete_item() - .table_name(LOG_TABLE_NAME) - .key( - LOG_TABLE_FIELD_LOG_ID, - AttributeValue::S(log_id.to_string()), - ) - .send() - .await - .map_err(|e| { - error!("DynamoDB client failed to remove log item"); - Error::AwsSdk(e.into()) - })?; - - Ok(()) - } } diff --git a/services/terraform/modules/shared/dynamodb.tf b/services/terraform/modules/shared/dynamodb.tf --- a/services/terraform/modules/shared/dynamodb.tf +++ b/services/terraform/modules/shared/dynamodb.tf @@ -41,7 +41,7 @@ attribute { name = "logID" - type = "S" + type = "N" } }