diff --git a/services/backup/src/Constants.h b/services/backup/src/Constants.h index 6c6f988a0..eebef98ac 100644 --- a/services/backup/src/Constants.h +++ b/services/backup/src/Constants.h @@ -1,26 +1,26 @@ #pragma once #include "GlobalTools.h" #include "Tools.h" #include namespace comm { namespace network { const std::string LOG_TABLE_NAME = tools::decorateTableName("backup-service-log"); const std::string BACKUP_TABLE_NAME = tools::decorateTableName("backup-service-backup"); // This has to be smaller than GRPC_CHUNK_SIZE_LIMIT because we need to // recognize if we may receive multiple chunks or just one. If it was larger // than the chunk limit, once we get the amount of data of size equal to the // limit, we wouldn't know if we should put this in the database right away or // wait for more data. -// 400KB limit (KB, not KiB, that's why it's 1000 not 1024) - +// 400KiB limit (in docs there is KB but they mean KiB) - // https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html -const size_t LOG_DATA_SIZE_DATABASE_LIMIT = 400 * 1000; +const size_t LOG_DATA_SIZE_DATABASE_LIMIT = 1024 * 400; } // namespace network } // namespace comm diff --git a/services/commtest/tests/backup_test.rs b/services/commtest/tests/backup_test.rs index 1dd9d08b3..b5d1ef163 100644 --- a/services/commtest/tests/backup_test.rs +++ b/services/commtest/tests/backup_test.rs @@ -1,125 +1,168 @@ #[path = "./backup/add_attachments.rs"] mod add_attachments; #[path = "./backup/backup_utils.rs"] mod backup_utils; #[path = "./backup/create_new_backup.rs"] mod create_new_backup; #[path = "./backup/pull_backup.rs"] mod pull_backup; #[path = "./backup/send_log.rs"] mod send_log; #[path = "./lib/tools.rs"] mod tools; use backup_utils::{BackupData, Item}; use bytesize::ByteSize; use tools::Error; use backup_utils::BackupServiceClient; #[tokio::test] async fn backup_test() -> Result<(), Error> { let mut client = BackupServiceClient::connect("http://localhost:50052").await?; let mut backup_data = BackupData { user_id: "user0000".to_string(), device_id: "device0000".to_string(), backup_item: Item::new( String::new(), vec![ByteSize::mib(1).as_u64() as usize; 6], vec![ + "holder0".to_string(), "holder1".to_string(), "holder2".to_string(), - "holder3".to_string(), ], ), log_items: vec![ - Item::new(String::new(), vec![ByteSize::b(100).as_u64() as usize], vec!["holder1".to_string()]), + // the item that almost hits the DB limit, we're going to later add a long + // list of attachments, so that causes it to exceed the limit. + // In this case its data should be moved to the S3 Item::new( String::new(), - vec![ByteSize::kb(400).as_u64() as usize], - vec!["holder2".to_string(), "holder3".to_string()], + vec![ + tools::get_dynamo_db_item_size_limit() + - ByteSize::b(100).as_u64() as usize, + ], + vec!["holder0".to_string(), "holder1".to_string()], + ), + // just a small item + Item::new( + String::new(), + vec![ByteSize::b(100).as_u64() as usize], + vec!["holder0".to_string()], ), + // a big item that should be placed in the S3 right away Item::new( String::new(), - vec![tools::get_grpc_chunk_size_limit(), tools::get_grpc_chunk_size_limit()], vec![ + tools::get_grpc_chunk_size_limit(), + tools::get_grpc_chunk_size_limit(), + ], + vec![ + "holder0".to_string(), "holder1".to_string(), "holder2".to_string(), - "holder3".to_string(), ], ), ], }; backup_data.backup_item.id = create_new_backup::run(&mut client, &backup_data).await?; println!("backup id in main: {}", backup_data.backup_item.id); add_attachments::run(&mut client, &backup_data, None).await?; for log_index in 0..backup_data.log_items.len() { backup_data.log_items[log_index].id = send_log::run(&mut client, &backup_data, log_index).await?; add_attachments::run(&mut client, &backup_data, Some(log_index)).await?; } let result = pull_backup::run(&mut client, &backup_data).await?; // check backup size let expected: usize = backup_data.backup_item.chunks_sizes.iter().sum(); let from_result: usize = result.backup_item.chunks_sizes.iter().sum(); assert_eq!( from_result, expected, "backup sizes do not match, expected {}, got {}", - expected, - from_result + expected, from_result ); // check backup attachments let expected: usize = backup_data.backup_item.attachments_holders.len(); let from_result: usize = result.backup_item.attachments_holders.len(); assert_eq!( from_result, expected, "backup: number of attachments holders do not match, expected {}, got {}", - expected, - from_result + expected, from_result ); // check number of logs let expected: usize = backup_data.log_items.len(); let from_result: usize = result.log_items.len(); assert_eq!( expected, from_result, "number of logs do not match, expected {}, got {}", - expected, - from_result + expected, from_result ); // check log sizes for i in 0..backup_data.log_items.len() { let expected: usize = backup_data.log_items[i].chunks_sizes.iter().sum(); let from_result: usize = result.log_items[i].chunks_sizes.iter().sum(); assert_eq!( from_result, expected, "log number {} sizes do not match, expected {}, got {}", - i, - expected, - from_result + i, expected, from_result ); } // check logs attachments for i in 0..backup_data.log_items.len() { let expected: usize = backup_data.log_items[i].attachments_holders.len(); let from_result: usize = result.log_items[i].attachments_holders.len(); assert_eq!( from_result, expected, "log {}: number of attachments holders do not match, expected {}, got {}", + i, expected, from_result + ); + } + + // push so many attachments that the log item's data will have to be moved + // from the db to the s3 + let mut attachments_size = 0; + let mut i = backup_data.log_items[0].attachments_holders.len(); + let mut new_attachments: Vec = vec![]; + while attachments_size < 500 { + let att = format!("holder{}", i); + attachments_size += att.len(); + new_attachments.push(att); + i += 1; + } + + let mut old_attachments = + backup_data.log_items[0].attachments_holders.clone(); + backup_data.log_items[0].attachments_holders = new_attachments; + add_attachments::run(&mut client, &backup_data, Some(0)).await?; + backup_data.log_items[0] + .attachments_holders + .append(&mut old_attachments); + let result = pull_backup::run(&mut client, &backup_data).await?; + // check logs attachments + for i in 0..backup_data.log_items.len() { + let expected: usize = backup_data.log_items[i].attachments_holders.len(); + let from_result: usize = result.log_items[i].attachments_holders.len(); + assert_eq!( + from_result, + expected, + "after attachment add: log {}: number of attachments holders do not match, + expected {}, got {}", i, expected, from_result ); } Ok(()) } diff --git a/services/commtest/tests/lib/tools.rs b/services/commtest/tests/lib/tools.rs index ade4e4d2d..8e9775a5a 100644 --- a/services/commtest/tests/lib/tools.rs +++ b/services/commtest/tests/lib/tools.rs @@ -1,29 +1,37 @@ use bytesize::ByteSize; #[allow(dead_code)] -pub fn generate_nbytes(number_of_bytes: usize, predefined_byte_value: Option) -> Vec { +pub fn generate_nbytes( + number_of_bytes: usize, + predefined_byte_value: Option, +) -> Vec { let byte_value = predefined_byte_value.unwrap_or(b'A'); return vec![byte_value; number_of_bytes]; } #[derive( Debug, derive_more::Display, derive_more::From, derive_more::Error, )] pub enum Error { #[display(...)] Proto(std::io::Error), #[display(...)] Tonic(tonic::transport::Error), #[display(...)] TonicStatus(tonic::Status), } +#[allow(dead_code)] +pub fn get_dynamo_db_item_size_limit() -> usize { + ByteSize::kib(400).as_u64() as usize +} + pub const GRPC_METADATA_SIZE_BYTES: usize = 5; #[allow(dead_code)] pub fn get_grpc_chunk_size_limit() -> usize { (ByteSize::mib(4).as_u64() as usize) - GRPC_METADATA_SIZE_BYTES } #[allow(dead_code)] pub const ATTACHMENT_DELIMITER: &str = ";";