diff --git a/services/terraform/dynamodb-test.tf b/services/terraform/dynamodb-test.tf index f6714b735..76d88bb78 100644 --- a/services/terraform/dynamodb-test.tf +++ b/services/terraform/dynamodb-test.tf @@ -1,164 +1,156 @@ resource "aws_dynamodb_table" "backup-service-backup-test" { name = "backup-service-backup-test" hash_key = "userID" range_key = "backupID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "backupID" type = "S" } attribute { name = "created" type = "S" } global_secondary_index { name = "userID-created-index" hash_key = "userID" range_key = "created" write_capacity = 10 read_capacity = 10 projection_type = "INCLUDE" non_key_attributes = ["recoveryData"] } } resource "aws_dynamodb_table" "backup-service-log-test" { name = "backup-service-log-test" hash_key = "backupID" range_key = "logID" write_capacity = 10 read_capacity = 10 attribute { name = "backupID" type = "S" } attribute { name = "logID" type = "S" } } resource "aws_dynamodb_table" "blob-service-blob-test" { name = "blob-service-blob-test" hash_key = "blobHash" write_capacity = 10 read_capacity = 10 attribute { name = "blobHash" type = "S" } } resource "aws_dynamodb_table" "blob-service-reverse-index-test" { name = "blob-service-reverse-index-test" hash_key = "holder" write_capacity = 10 read_capacity = 10 attribute { name = "holder" type = "S" } attribute { name = "blobHash" type = "S" } global_secondary_index { name = "blobHash-index" hash_key = "blobHash" write_capacity = 10 read_capacity = 10 projection_type = "ALL" } } resource "aws_dynamodb_table" "tunnelbroker-device-sessions-test" { name = "tunnelbroker-device-sessions-test" hash_key = "SessionID" write_capacity = 10 read_capacity = 10 attribute { name = "SessionID" type = "S" } ttl { attribute_name = "Expire" enabled = true } } resource "aws_dynamodb_table" "tunnelbroker-verification-messages-test" { name = "tunnelbroker-verification-messages-test" hash_key = "DeviceID" write_capacity = 10 read_capacity = 10 attribute { name = "DeviceID" type = "S" } ttl { attribute_name = "Expire" enabled = true } } resource "aws_dynamodb_table" "tunnelbroker-public-keys-test" { name = "tunnelbroker-public-keys-test" hash_key = "DeviceID" write_capacity = 10 read_capacity = 10 attribute { name = "DeviceID" type = "S" } } resource "aws_dynamodb_table" "tunnelbroker-messages-test" { name = "tunnelbroker-messages-test" hash_key = "ToDeviceID" range_key = "MessageID" write_capacity = 10 read_capacity = 10 attribute { name = "MessageID" type = "S" } attribute { name = "ToDeviceID" type = "S" } - global_secondary_index { - name = "ToDeviceID-index" - hash_key = "ToDeviceID" - write_capacity = 10 - read_capacity = 10 - projection_type = "ALL" - } - ttl { attribute_name = "Expire" enabled = true } } diff --git a/services/terraform/dynamodb.tf b/services/terraform/dynamodb.tf index 88f9373b1..255f8b4d9 100644 --- a/services/terraform/dynamodb.tf +++ b/services/terraform/dynamodb.tf @@ -1,220 +1,212 @@ resource "aws_dynamodb_table" "backup-service-backup" { name = "backup-service-backup" hash_key = "userID" range_key = "backupID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "backupID" type = "S" } attribute { name = "created" type = "S" } global_secondary_index { name = "userID-created-index" hash_key = "userID" range_key = "created" write_capacity = 10 read_capacity = 10 projection_type = "INCLUDE" non_key_attributes = ["recoveryData"] } } resource "aws_dynamodb_table" "backup-service-log" { name = "backup-service-log" hash_key = "backupID" range_key = "logID" write_capacity = 10 read_capacity = 10 attribute { name = "backupID" type = "S" } attribute { name = "logID" type = "S" } } resource "aws_dynamodb_table" "blob-service-blob" { name = "blob-service-blob" hash_key = "blobHash" write_capacity = 10 read_capacity = 10 attribute { name = "blobHash" type = "S" } } resource "aws_dynamodb_table" "blob-service-reverse-index" { name = "blob-service-reverse-index" hash_key = "holder" write_capacity = 10 read_capacity = 10 attribute { name = "holder" type = "S" } attribute { name = "blobHash" type = "S" } global_secondary_index { name = "blobHash-index" hash_key = "blobHash" write_capacity = 10 read_capacity = 10 projection_type = "ALL" } } resource "aws_dynamodb_table" "tunnelbroker-device-sessions" { name = "tunnelbroker-device-sessions" hash_key = "SessionID" write_capacity = 10 read_capacity = 10 attribute { name = "SessionID" type = "S" } ttl { attribute_name = "Expire" enabled = true } } resource "aws_dynamodb_table" "tunnelbroker-verification-messages" { name = "tunnelbroker-verification-messages" hash_key = "DeviceID" write_capacity = 10 read_capacity = 10 attribute { name = "DeviceID" type = "S" } ttl { attribute_name = "Expire" enabled = true } } resource "aws_dynamodb_table" "tunnelbroker-public-keys" { name = "tunnelbroker-public-keys" hash_key = "DeviceID" write_capacity = 10 read_capacity = 10 attribute { name = "DeviceID" type = "S" } } resource "aws_dynamodb_table" "tunnelbroker-messages" { name = "tunnelbroker-messages" hash_key = "ToDeviceID" range_key = "MessageID" write_capacity = 10 read_capacity = 10 attribute { name = "MessageID" type = "S" } attribute { name = "ToDeviceID" type = "S" } - global_secondary_index { - name = "ToDeviceID-index" - hash_key = "ToDeviceID" - write_capacity = 10 - read_capacity = 10 - projection_type = "ALL" - } - ttl { attribute_name = "Expire" enabled = true } } resource "aws_dynamodb_table" "identity-users" { name = "identity-users" hash_key = "userID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "username" type = "S" } attribute { name = "walletAddress" type = "S" } global_secondary_index { name = "username-index" hash_key = "username" write_capacity = 10 read_capacity = 10 projection_type = "KEYS_ONLY" } global_secondary_index { name = "walletAddress-index" hash_key = "walletAddress" write_capacity = 10 read_capacity = 10 projection_type = "KEYS_ONLY" } } resource "aws_dynamodb_table" "identity-tokens" { name = "identity-tokens" hash_key = "userID" range_key = "deviceID" write_capacity = 10 read_capacity = 10 attribute { name = "userID" type = "S" } attribute { name = "deviceID" type = "S" } } diff --git a/services/tunnelbroker/src/Database/DatabaseManager.cpp b/services/tunnelbroker/src/Database/DatabaseManager.cpp index 0bdcbd30a..4795959ee 100644 --- a/services/tunnelbroker/src/Database/DatabaseManager.cpp +++ b/services/tunnelbroker/src/Database/DatabaseManager.cpp @@ -1,272 +1,270 @@ #include "DatabaseManager.h" #include "DynamoDBTools.h" #include "GlobalTools.h" namespace comm { namespace network { namespace database { DatabaseManager &DatabaseManager::getInstance() { static DatabaseManager instance; return instance; } bool DatabaseManager::isTableAvailable(const std::string &tableName) { Aws::DynamoDB::Model::DescribeTableRequest request; request.SetTableName(tableName); // Check table availability by invoking DescribeTable const Aws::DynamoDB::Model::DescribeTableOutcome &result = getDynamoDBClient()->DescribeTable(request); return result.IsSuccess(); } void DatabaseManager::putSessionItem(const DeviceSessionItem &item) { Aws::DynamoDB::Model::PutItemRequest request; request.SetTableName(item.getTableName()); request.AddItem( DeviceSessionItem::FIELD_SESSION_ID, Aws::DynamoDB::Model::AttributeValue(item.getSessionID())); request.AddItem( DeviceSessionItem::FIELD_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(item.getDeviceID())); request.AddItem( DeviceSessionItem::FIELD_PUBKEY, Aws::DynamoDB::Model::AttributeValue(item.getPubKey())); request.AddItem( DeviceSessionItem::FIELD_NOTIFY_TOKEN, Aws::DynamoDB::Model::AttributeValue(item.getNotifyToken())); request.AddItem( DeviceSessionItem::FIELD_DEVICE_TYPE, Aws::DynamoDB::Model::AttributeValue(item.getDeviceType())); request.AddItem( DeviceSessionItem::FIELD_APP_VERSION, Aws::DynamoDB::Model::AttributeValue(item.getAppVersion())); request.AddItem( DeviceSessionItem::FIELD_DEVICE_OS, Aws::DynamoDB::Model::AttributeValue(item.getDeviceOs())); request.AddItem( DeviceSessionItem::FIELD_CHECKPOINT_TIME, Aws::DynamoDB::Model::AttributeValue( std::to_string(item.getCheckpointTime()))); request.AddItem( DeviceSessionItem::FIELD_EXPIRE, Aws::DynamoDB::Model::AttributeValue(std::to_string( static_cast(std::time(0)) + SESSION_RECORD_TTL))); this->innerPutItem(std::make_shared(item), request); } std::shared_ptr DatabaseManager::findSessionItem(const std::string &sessionID) { Aws::DynamoDB::Model::GetItemRequest request; request.AddKey( DeviceSessionItem::FIELD_SESSION_ID, Aws::DynamoDB::Model::AttributeValue(sessionID)); return std::move(this->innerFindItem(request)); } void DatabaseManager::removeSessionItem(const std::string &sessionID) { std::shared_ptr item = this->findSessionItem(sessionID); if (item == nullptr) { return; } this->innerRemoveItem(*item); } void DatabaseManager::putSessionSignItem(const SessionSignItem &item) { Aws::DynamoDB::Model::PutItemRequest request; request.SetTableName(item.getTableName()); request.AddItem( SessionSignItem::FIELD_SESSION_VERIFICATION, Aws::DynamoDB::Model::AttributeValue(item.getSign())); request.AddItem( SessionSignItem::FIELD_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(item.getDeviceID())); request.AddItem( SessionSignItem::FIELD_EXPIRE, Aws::DynamoDB::Model::AttributeValue(std::to_string( static_cast(std::time(0)) + SESSION_SIGN_RECORD_TTL))); this->innerPutItem(std::make_shared(item), request); } std::shared_ptr DatabaseManager::findSessionSignItem(const std::string &deviceID) { Aws::DynamoDB::Model::GetItemRequest request; request.AddKey( SessionSignItem::FIELD_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(deviceID)); return std::move(this->innerFindItem(request)); } void DatabaseManager::removeSessionSignItem(const std::string &deviceID) { std::shared_ptr item = this->findSessionSignItem(deviceID); if (item == nullptr) { return; } this->innerRemoveItem(*item); } void DatabaseManager::putPublicKeyItem(const PublicKeyItem &item) { Aws::DynamoDB::Model::PutItemRequest request; request.SetTableName(item.getTableName()); request.AddItem( PublicKeyItem::FIELD_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(item.getDeviceID())); request.AddItem( PublicKeyItem::FIELD_PUBLIC_KEY, Aws::DynamoDB::Model::AttributeValue(item.getPublicKey())); this->innerPutItem(std::make_shared(item), request); } std::shared_ptr DatabaseManager::findPublicKeyItem(const std::string &deviceID) { Aws::DynamoDB::Model::GetItemRequest request; request.AddKey( PublicKeyItem::FIELD_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(deviceID)); return std::move(this->innerFindItem(request)); } void DatabaseManager::removePublicKeyItem(const std::string &deviceID) { std::shared_ptr item = this->findPublicKeyItem(deviceID); if (item == nullptr) { return; } this->innerRemoveItem(*item); } template T DatabaseManager::populatePutRequestFromMessageItem( T &putRequest, const MessageItem &item) { putRequest.AddItem( MessageItem::FIELD_MESSAGE_ID, Aws::DynamoDB::Model::AttributeValue(item.getMessageID())); putRequest.AddItem( MessageItem::FIELD_FROM_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(item.getFromDeviceID())); putRequest.AddItem( MessageItem::FIELD_TO_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(item.getToDeviceID())); putRequest.AddItem( MessageItem::FIELD_PAYLOAD, Aws::DynamoDB::Model::AttributeValue(item.getPayload())); putRequest.AddItem( MessageItem::FIELD_BLOB_HASHES, Aws::DynamoDB::Model::AttributeValue(item.getBlobHashes())); putRequest.AddItem( MessageItem::FIELD_EXPIRE, Aws::DynamoDB::Model::AttributeValue(std::to_string( static_cast(std::time(0) + MESSAGE_RECORD_TTL)))); putRequest.AddItem( MessageItem::FIELD_CREATED_AT, Aws::DynamoDB::Model::AttributeValue( std::to_string(tools::getCurrentTimestamp()))); return putRequest; } void DatabaseManager::putMessageItem(const MessageItem &item) { Aws::DynamoDB::Model::PutItemRequest request; request = this->populatePutRequestFromMessageItem(request, item); request.SetTableName(item.getTableName()); this->innerPutItem(std::make_shared(item), request); } void DatabaseManager::putMessageItemsByBatch( std::vector &messageItems) { std::vector writeRequests; for (MessageItem &messageItem : messageItems) { Aws::DynamoDB::Model::PutRequest putRequest; putRequest = this->populatePutRequestFromMessageItem(putRequest, messageItem); Aws::DynamoDB::Model::WriteRequest writeRequest; writeRequest.SetPutRequest(putRequest); writeRequests.push_back(writeRequest); } this->innerBatchWriteItem( MESSAGES_TABLE_NAME, DYNAMODB_MAX_BATCH_ITEMS, DYNAMODB_BACKOFF_FIRST_RETRY_DELAY, DYNAMODB_MAX_BACKOFF_TIME, writeRequests); } std::shared_ptr DatabaseManager::findMessageItem( const std::string &toDeviceID, const std::string &messageID) { Aws::DynamoDB::Model::GetItemRequest request; request.AddKey( MessageItem::FIELD_TO_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(toDeviceID)); request.AddKey( MessageItem::FIELD_MESSAGE_ID, Aws::DynamoDB::Model::AttributeValue(messageID)); return std::move(this->innerFindItem(request)); } std::vector> DatabaseManager::findMessageItemsByReceiver(const std::string &toDeviceID) { std::vector> result; Aws::DynamoDB::Model::QueryRequest req; req.SetTableName(MessageItem().getTableName()); req.SetKeyConditionExpression( MessageItem::FIELD_TO_DEVICE_ID + " = :valueToMatch"); AttributeValues attributeValues; attributeValues.emplace(":valueToMatch", toDeviceID); req.SetExpressionAttributeValues(attributeValues); - req.SetIndexName(MessageItem::INDEX_TO_DEVICE_ID); - const Aws::DynamoDB::Model::QueryOutcome &outcome = getDynamoDBClient()->Query(req); if (!outcome.IsSuccess()) { throw std::runtime_error(outcome.GetError().GetMessage()); } const Aws::Vector &items = outcome.GetResult().GetItems(); for (auto &item : items) { result.push_back(std::make_shared(item)); } return result; } void DatabaseManager::removeMessageItem( const std::string &toDeviceID, const std::string &messageID) { std::shared_ptr item = this->findMessageItem(toDeviceID, messageID); if (item == nullptr) { return; } this->innerRemoveItem(*item); } void DatabaseManager::removeMessageItemsByIDsForDeviceID( std::vector &messageIDs, const std::string &toDeviceID) { std::vector writeRequests; for (std::string &messageID : messageIDs) { Aws::DynamoDB::Model::DeleteRequest deleteRequest; deleteRequest.AddKey( MessageItem::FIELD_TO_DEVICE_ID, Aws::DynamoDB::Model::AttributeValue(toDeviceID)); deleteRequest.AddKey( MessageItem::FIELD_MESSAGE_ID, Aws::DynamoDB::Model::AttributeValue(messageID)); Aws::DynamoDB::Model::WriteRequest currentWriteRequest; currentWriteRequest.SetDeleteRequest(deleteRequest); writeRequests.push_back(currentWriteRequest); } this->innerBatchWriteItem( MESSAGES_TABLE_NAME, DYNAMODB_MAX_BATCH_ITEMS, DYNAMODB_BACKOFF_FIRST_RETRY_DELAY, DYNAMODB_MAX_BACKOFF_TIME, writeRequests); } } // namespace database } // namespace network } // namespace comm diff --git a/services/tunnelbroker/src/Database/MessageItem.cpp b/services/tunnelbroker/src/Database/MessageItem.cpp index 5cd1442d2..62417cc71 100644 --- a/services/tunnelbroker/src/Database/MessageItem.cpp +++ b/services/tunnelbroker/src/Database/MessageItem.cpp @@ -1,110 +1,109 @@ #include "MessageItem.h" #include "ConfigManager.h" #include "Tools.h" #include namespace comm { namespace network { namespace database { const std::string MessageItem::FIELD_MESSAGE_ID = "MessageID"; const std::string MessageItem::FIELD_FROM_DEVICE_ID = "FromDeviceID"; const std::string MessageItem::FIELD_TO_DEVICE_ID = "ToDeviceID"; const std::string MessageItem::FIELD_PAYLOAD = "Payload"; const std::string MessageItem::FIELD_BLOB_HASHES = "BlobHashes"; const std::string MessageItem::FIELD_EXPIRE = "Expire"; const std::string MessageItem::FIELD_CREATED_AT = "CreatedAt"; -const std::string MessageItem::INDEX_TO_DEVICE_ID = "ToDeviceID-index"; MessageItem::MessageItem( const std::string messageID, const std::string fromDeviceID, const std::string toDeviceID, const std::string payload, const std::string blobHashes) : messageID(messageID), fromDeviceID(fromDeviceID), toDeviceID(toDeviceID), payload(payload), blobHashes(blobHashes) { this->validate(); } MessageItem::MessageItem(const AttributeValues &itemFromDB) { this->assignItemFromDatabase(itemFromDB); } void MessageItem::validate() const { if (!tools::validateDeviceID(this->fromDeviceID)) { throw std::runtime_error("Error: FromDeviceID format is wrong."); } if (!tools::validateDeviceID(this->toDeviceID)) { throw std::runtime_error("Error: ToDeviceID format is wrong."); } tools::checkIfNotEmpty("messageID", this->messageID); } void MessageItem::assignItemFromDatabase(const AttributeValues &itemFromDB) { try { this->messageID = itemFromDB.at(MessageItem::FIELD_MESSAGE_ID).GetS(); this->fromDeviceID = itemFromDB.at(MessageItem::FIELD_FROM_DEVICE_ID).GetS(); this->toDeviceID = itemFromDB.at(MessageItem::FIELD_TO_DEVICE_ID).GetS(); this->payload = itemFromDB.at(MessageItem::FIELD_PAYLOAD).GetS(); this->blobHashes = itemFromDB.at(MessageItem::FIELD_BLOB_HASHES).GetS(); this->expire = std::stoull(itemFromDB.at(MessageItem::FIELD_EXPIRE).GetS()); this->createdAt = std::stoull(itemFromDB.at(MessageItem::FIELD_CREATED_AT).GetS()); } catch (const std::exception &e) { throw std::runtime_error( "Got an exception at MessageItem: " + std::string(e.what())); } this->validate(); } std::string MessageItem::getTableName() const { return config::ConfigManager::getInstance().getParameter( config::ConfigManager::OPTION_DYNAMODB_MESSAGES_TABLE); } PrimaryKeyDescriptor MessageItem::getPrimaryKeyDescriptor() const { return PrimaryKeyDescriptor( MessageItem::FIELD_TO_DEVICE_ID, MessageItem::FIELD_MESSAGE_ID); } PrimaryKeyValue MessageItem::getPrimaryKeyValue() const { return PrimaryKeyValue(this->toDeviceID, this->messageID); } std::string MessageItem::getMessageID() const { return this->messageID; } std::string MessageItem::getFromDeviceID() const { return this->fromDeviceID; } std::string MessageItem::getToDeviceID() const { return this->toDeviceID; } std::string MessageItem::getPayload() const { return this->payload; } std::string MessageItem::getBlobHashes() const { return this->blobHashes; } uint64_t MessageItem::getExpire() const { return this->expire; } uint64_t MessageItem::getCreatedAt() const { return this->createdAt; } } // namespace database } // namespace network } // namespace comm diff --git a/services/tunnelbroker/src/Database/MessageItem.h b/services/tunnelbroker/src/Database/MessageItem.h index 7ad796b05..9f4a253aa 100644 --- a/services/tunnelbroker/src/Database/MessageItem.h +++ b/services/tunnelbroker/src/Database/MessageItem.h @@ -1,57 +1,56 @@ #pragma once #include "Item.h" #include namespace comm { namespace network { namespace database { class MessageItem : public Item { std::string messageID; std::string fromDeviceID; std::string toDeviceID; std::string payload; std::string blobHashes; uint64_t expire; uint64_t createdAt; void validate() const override; public: static const std::string FIELD_MESSAGE_ID; static const std::string FIELD_FROM_DEVICE_ID; static const std::string FIELD_TO_DEVICE_ID; static const std::string FIELD_PAYLOAD; static const std::string FIELD_BLOB_HASHES; static const std::string FIELD_EXPIRE; static const std::string FIELD_CREATED_AT; - static const std::string INDEX_TO_DEVICE_ID; PrimaryKeyDescriptor getPrimaryKeyDescriptor() const override; PrimaryKeyValue getPrimaryKeyValue() const override; std::string getTableName() const override; std::string getMessageID() const; std::string getFromDeviceID() const; std::string getToDeviceID() const; std::string getPayload() const; std::string getBlobHashes() const; uint64_t getExpire() const; uint64_t getCreatedAt() const; MessageItem() { } MessageItem( const std::string messageID, const std::string fromDeviceID, const std::string toDeviceID, const std::string payload, const std::string blobHashes); MessageItem(const AttributeValues &itemFromDB); void assignItemFromDatabase(const AttributeValues &itemFromDB) override; }; } // namespace database } // namespace network } // namespace comm