diff --git a/services/blob/src/constants.rs b/services/blob/src/constants.rs index 0c2cac831..d02f45e3e 100644 --- a/services/blob/src/constants.rs +++ b/services/blob/src/constants.rs @@ -1,52 +1,74 @@ // Assorted constants pub const DEFAULT_GRPC_PORT: u16 = 50051; pub const DEFAULT_HTTP_PORT: u16 = 51001; pub const LOCALSTACK_URL: &str = "http://localstack:4566"; pub const MPSC_CHANNEL_BUFFER_CAPACITY: usize = 1; /// 4MB limit /// /// WARNING: use keeping in mind that grpc adds its own headers to messages /// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md /// so the message that actually is being sent over the network looks like this /// ``` /// [Compressed-Flag] [Message-Length] [Message] /// [Compressed-Flag] 1 byte - added by grpc /// [Message-Length] 4 bytes - added by grpc /// [Message] N bytes - actual data /// ``` /// so for every message we get 5 additional bytes of data /// as [mentioned here](https://github.com/grpc/grpc/issues/15734#issuecomment-396962671), /// gRPC stream may contain more than one message pub const GRPC_CHUNK_SIZE_LIMIT: u64 = 4 * 1024 * 1024; /// See [`GRPC_CHUNK_SIZE_LIMIT`] description for details pub const GRPC_METADATA_SIZE_PER_MESSAGE: u64 = 5; // HTTP constants pub const BLOB_DOWNLOAD_CHUNK_SIZE: u64 = 5 * 1024 * 1024; // DynamoDB constants +pub mod db { + /// Reserved holder value that indicates the row is a blob item + pub const BLOB_ITEM_ROW_HOLDER_VALUE: &str = "_"; + + pub const BLOB_TABLE_NAME: &str = "blob-service-blobs"; + pub const BLOB_PARTITION_KEY: &str = ATTR_BLOB_HASH; + pub const BLOB_SORT_KEY: &str = ATTR_HOLDER; + + pub const UNCHECKED_INDEX_NAME: &str = "unchecked-index"; + pub const UNCHECKED_INDEX_PARTITION_KEY: &str = ATTR_UNCHECKED; + pub const UNCHECKED_INDEX_SORT_KEY: &str = ATTR_LAST_MODIFIED; + + /// attribute names + pub const ATTR_BLOB_HASH: &str = "blob_hash"; + pub const ATTR_HOLDER: &str = "holder"; + pub const ATTR_CREATED_AT: &str = "created_at"; + pub const ATTR_LAST_MODIFIED: &str = "last_modified"; + pub const ATTR_S3_PATH: &str = "s3_path"; + pub const ATTR_UNCHECKED: &str = "unchecked"; +} + +// old DynamoDB constants pub const BLOB_TABLE_NAME: &str = "blob-service-blob"; pub const BLOB_TABLE_BLOB_HASH_FIELD: &str = "blobHash"; pub const BLOB_TABLE_S3_PATH_FIELD: &str = "s3Path"; pub const BLOB_TABLE_CREATED_FIELD: &str = "created"; pub const BLOB_REVERSE_INDEX_TABLE_NAME: &str = "blob-service-reverse-index"; pub const BLOB_REVERSE_INDEX_TABLE_HOLDER_FIELD: &str = "holder"; pub const BLOB_REVERSE_INDEX_TABLE_BLOB_HASH_FIELD: &str = "blobHash"; pub const BLOB_REVERSE_INDEX_TABLE_HASH_INDEX_NAME: &str = "blobHash-index"; // Environment variables pub const SANDBOX_ENV_VAR: &str = "COMM_SERVICES_SANDBOX"; pub const LOG_LEVEL_ENV_VAR: &str = tracing_subscriber::filter::EnvFilter::DEFAULT_ENV; // S3 constants pub const BLOB_S3_BUCKET_NAME: &str = "commapp-blob"; pub const S3_MULTIPART_UPLOAD_MINIMUM_CHUNK_SIZE: u64 = 5 * 1024 * 1024; diff --git a/services/terraform/modules/shared/dynamodb.tf b/services/terraform/modules/shared/dynamodb.tf index ac09fd2a0..0bf38f79b 100644 --- a/services/terraform/modules/shared/dynamodb.tf +++ b/services/terraform/modules/shared/dynamodb.tf @@ -1,241 +1,277 @@ resource "aws_dynamodb_table" "backup-service-backup" { name = "backup-service-backup" hash_key = "userID" range_key = "backupID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "backupID" type = "S" } attribute { name = "created" type = "S" } global_secondary_index { name = "userID-created-index" hash_key = "userID" range_key = "created" write_capacity = 10 read_capacity = 10 projection_type = "INCLUDE" non_key_attributes = ["recoveryData"] } } resource "aws_dynamodb_table" "backup-service-log" { name = "backup-service-log" hash_key = "backupID" range_key = "logID" write_capacity = 10 read_capacity = 10 attribute { name = "backupID" type = "S" } attribute { name = "logID" type = "S" } } resource "aws_dynamodb_table" "blob-service-blob" { name = "blob-service-blob" hash_key = "blobHash" write_capacity = 10 read_capacity = 10 attribute { name = "blobHash" type = "S" } } resource "aws_dynamodb_table" "blob-service-reverse-index" { name = "blob-service-reverse-index" hash_key = "holder" write_capacity = 10 read_capacity = 10 attribute { name = "holder" type = "S" } attribute { name = "blobHash" type = "S" } global_secondary_index { name = "blobHash-index" hash_key = "blobHash" write_capacity = 10 read_capacity = 10 projection_type = "ALL" } } +resource "aws_dynamodb_table" "blob-service-blobs" { + name = "blob-service-blobs" + hash_key = "blob_hash" + range_key = "holder" + billing_mode = "PAY_PER_REQUEST" + + attribute { + name = "blob_hash" + type = "S" + } + + attribute { + name = "holder" + type = "S" + } + + attribute { + name = "last_modified" + type = "N" + } + + attribute { + name = "unchecked" + type = "S" + } + + global_secondary_index { + name = "unchecked-index" + hash_key = "unchecked" + range_key = "last_modified" + + projection_type = "INCLUDE" + non_key_attributes = ["blob_hash", "holder"] + } +} + resource "aws_dynamodb_table" "tunnelbroker-undelivered-messages" { name = "tunnelbroker-undelivered-messages" hash_key = "deviceID" range_key = "createdAt" write_capacity = 10 read_capacity = 10 attribute { name = "deviceID" type = "S" } attribute { name = "createdAt" type = "N" } } resource "aws_dynamodb_table" "identity-users" { name = "identity-users" hash_key = "userID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "username" type = "S" } attribute { name = "walletAddress" type = "S" } global_secondary_index { name = "username-index" hash_key = "username" write_capacity = 10 read_capacity = 10 projection_type = "KEYS_ONLY" } global_secondary_index { name = "walletAddress-index" hash_key = "walletAddress" write_capacity = 10 read_capacity = 10 projection_type = "KEYS_ONLY" } } # Identity users with opaque_ke 2.0 credentials resource "aws_dynamodb_table" "identity-users-opaque2" { name = "identity-users-opaque2" hash_key = "userID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "username" type = "S" } attribute { name = "walletAddress" type = "S" } global_secondary_index { name = "username-index" hash_key = "username" write_capacity = 10 read_capacity = 10 projection_type = "KEYS_ONLY" } global_secondary_index { name = "walletAddress-index" hash_key = "walletAddress" write_capacity = 10 read_capacity = 10 projection_type = "KEYS_ONLY" } } resource "aws_dynamodb_table" "identity-tokens" { name = "identity-tokens" hash_key = "userID" range_key = "signingPublicKey" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "signingPublicKey" type = "S" } } resource "aws_dynamodb_table" "identity-nonces" { name = "identity-nonces" hash_key = "nonce" write_capacity = 10 read_capacity = 10 attribute { name = "nonce" type = "S" } } resource "aws_dynamodb_table" "identity-reserved-usernames" { name = "identity-reserved-usernames" hash_key = "username" write_capacity = 10 read_capacity = 10 attribute { name = "username" type = "S" } } resource "aws_dynamodb_table" "feature-flags" { name = "feature-flags" hash_key = "platform" range_key = "feature" billing_mode = "PAY_PER_REQUEST" attribute { name = "platform" type = "S" } attribute { name = "feature" type = "S" } }