Page MenuHomePhabricator
Files F2907798

D4324.id14317.diff
No OneTemporary
Actions

D4324.id14317.diff
View Options

diff --git a/services/backup/src/Constants.h b/services/backup/src/Constants.h
--- a/services/backup/src/Constants.h
+++ b/services/backup/src/Constants.h
@@ -18,9 +18,9 @@
// than the chunk limit, once we get the amount of data of size equal to the
// limit, we wouldn't know if we should put this in the database right away or
// wait for more data.
-// 400KB limit (KB, not KiB, that's why it's 1000 not 1024) -
+// 400KiB limit (in docs there is KB but they mean KiB) -
// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html
-const size_t LOG_DATA_SIZE_DATABASE_LIMIT = 400 * 1000;
+const size_t LOG_DATA_SIZE_DATABASE_LIMIT = 1024 * 400;
} // namespace network
} // namespace comm
diff --git a/services/commtest/tests/backup_test.rs b/services/commtest/tests/backup_test.rs
--- a/services/commtest/tests/backup_test.rs
+++ b/services/commtest/tests/backup_test.rs
@@ -29,25 +29,40 @@
String::new(),
vec![ByteSize::mib(1).as_u64() as usize; 6],
vec![
+ "holder0".to_string(),
"holder1".to_string(),
"holder2".to_string(),
- "holder3".to_string(),
],
),
log_items: vec![
- Item::new(String::new(), vec![ByteSize::b(100).as_u64() as usize], vec!["holder1".to_string()]),
+ // the item that almost hits the DB limit, we're going to later add a long
+ // list of attachments, so that causes it to exceed the limit.
+ // In this case its data should be moved to the S3
Item::new(
String::new(),
- vec![ByteSize::kb(400).as_u64() as usize],
- vec!["holder2".to_string(), "holder3".to_string()],
+ vec![
+ tools::get_dynamo_db_item_size_limit()
+ - ByteSize::b(100).as_u64() as usize,
+ ],
+ vec!["holder0".to_string(), "holder1".to_string()],
+ ),
+ // just a small item
+ Item::new(
+ String::new(),
+ vec![ByteSize::b(100).as_u64() as usize],
+ vec!["holder0".to_string()],
),
+ // a big item that should be placed in the S3 right away
Item::new(
String::new(),
- vec![tools::get_grpc_chunk_size_limit(), tools::get_grpc_chunk_size_limit()],
vec![
+ tools::get_grpc_chunk_size_limit(),
+ tools::get_grpc_chunk_size_limit(),
+ ],
+ vec![
+ "holder0".to_string(),
"holder1".to_string(),
"holder2".to_string(),
- "holder3".to_string(),
],
),
],
@@ -72,8 +87,7 @@
assert_eq!(
from_result, expected,
"backup sizes do not match, expected {}, got {}",
- expected,
- from_result
+ expected, from_result
);
// check backup attachments
@@ -82,8 +96,7 @@
assert_eq!(
from_result, expected,
"backup: number of attachments holders do not match, expected {}, got {}",
- expected,
- from_result
+ expected, from_result
);
// check number of logs
@@ -92,8 +105,7 @@
assert_eq!(
expected, from_result,
"number of logs do not match, expected {}, got {}",
- expected,
- from_result
+ expected, from_result
);
// check log sizes
@@ -103,9 +115,7 @@
assert_eq!(
from_result, expected,
"log number {} sizes do not match, expected {}, got {}",
- i,
- expected,
- from_result
+ i, expected, from_result
);
}
// check logs attachments
@@ -115,6 +125,39 @@
assert_eq!(
from_result, expected,
"log {}: number of attachments holders do not match, expected {}, got {}",
+ i, expected, from_result
+ );
+ }
+
+ // push so many attachments that the log item's data will have to be moved
+ // from the db to the s3
+ let mut attachments_size = 0;
+ let mut i = backup_data.log_items[0].attachments_holders.len();
+ let mut new_attachments: Vec<String> = vec![];
+ while attachments_size < 500 {
+ let att = format!("holder{}", i);
+ attachments_size += att.len();
+ new_attachments.push(att);
+ i += 1;
+ }
+
+ let mut old_attachments =
+ backup_data.log_items[0].attachments_holders.clone();
+ backup_data.log_items[0].attachments_holders = new_attachments;
+ add_attachments::run(&mut client, &backup_data, Some(0)).await?;
+ backup_data.log_items[0]
+ .attachments_holders
+ .append(&mut old_attachments);
+ let result = pull_backup::run(&mut client, &backup_data).await?;
+ // check logs attachments
+ for i in 0..backup_data.log_items.len() {
+ let expected: usize = backup_data.log_items[i].attachments_holders.len();
+ let from_result: usize = result.log_items[i].attachments_holders.len();
+ assert_eq!(
+ from_result,
+ expected,
+ "after attachment add: log {}: number of attachments holders do not match,
+ expected {}, got {}",
i,
expected,
from_result
diff --git a/services/commtest/tests/lib/tools.rs b/services/commtest/tests/lib/tools.rs
--- a/services/commtest/tests/lib/tools.rs
+++ b/services/commtest/tests/lib/tools.rs
@@ -1,7 +1,10 @@
use bytesize::ByteSize;
#[allow(dead_code)]
-pub fn generate_nbytes(number_of_bytes: usize, predefined_byte_value: Option<u8>) -> Vec<u8> {
+pub fn generate_nbytes(
+ number_of_bytes: usize,
+ predefined_byte_value: Option<u8>,
+) -> Vec<u8> {
let byte_value = predefined_byte_value.unwrap_or(b'A');
return vec![byte_value; number_of_bytes];
}
@@ -18,6 +21,11 @@
TonicStatus(tonic::Status),
}
+#[allow(dead_code)]
+pub fn get_dynamo_db_item_size_limit() -> usize {
+ ByteSize::kib(400).as_u64() as usize
+}
+
pub const GRPC_METADATA_SIZE_BYTES: usize = 5;
#[allow(dead_code)]

File Metadata

Mime Type
text/plain
Expires
Mon, Oct 7, 12:17 PM (21 h, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2254394
Default Alt Text
D4324.id14317.diff (5 KB)

Event Timeline