diff --git a/services/blob/src/AwsS3Bucket.cpp b/services/blob/src/AwsS3Bucket.cpp --- a/services/blob/src/AwsS3Bucket.cpp +++ b/services/blob/src/AwsS3Bucket.cpp @@ -16,6 +16,8 @@ #include +#include + namespace comm { namespace network { @@ -23,6 +25,7 @@ } std::vector AwsS3Bucket::listObjects() const { + LOG(INFO) << "[AwsS3Bucket::listObjects]"; Aws::S3::Model::ListObjectsRequest request; request.SetBucket(this->name); std::vector result; @@ -41,6 +44,7 @@ } bool AwsS3Bucket::isAvailable() const { + LOG(INFO) << "[AwsS3Bucket::isAvailable]"; Aws::S3::Model::HeadBucketRequest headRequest; headRequest.SetBucket(this->name); Aws::S3::Model::HeadBucketOutcome outcome = @@ -49,6 +53,7 @@ } size_t AwsS3Bucket::getObjectSize(const std::string &objectName) const { + LOG(INFO) << "[AwsS3Bucket::getObjectSize] object name " << objectName; Aws::S3::Model::HeadObjectRequest headRequest; headRequest.SetBucket(this->name); headRequest.SetKey(objectName); @@ -63,6 +68,8 @@ void AwsS3Bucket::renameObject( const std::string ¤tName, const std::string &newName) { + LOG(INFO) << "[AwsS3Bucket::renameObject] current name " << currentName; + LOG(INFO) << "[AwsS3Bucket::renameObject] new name " << newName; Aws::S3::Model::CopyObjectRequest copyRequest; copyRequest.SetCopySource(this->name + "/" + currentName); copyRequest.SetKey(newName); @@ -80,6 +87,7 @@ void AwsS3Bucket::writeObject( const std::string &objectName, const std::string &data) { + LOG(INFO) << "[AwsS3Bucket::writeObject] object name " << objectName; // we don't have to handle multiple write here because the GRPC limit is 4MB // and minimum size of data to perform multipart upload is 5MB Aws::S3::Model::PutObjectRequest request; @@ -99,6 +107,7 @@ } std::string AwsS3Bucket::getObjectData(const std::string &objectName) const { + LOG(INFO) << "[AwsS3Bucket::getObjectData] object name " << objectName; Aws::S3::Model::GetObjectRequest request; request.SetBucket(this->name); request.SetKey(objectName); @@ -130,6 +139,8 @@ const std::string &objectName, const std::function &callback, const size_t chunkSize) const { + LOG(INFO) << "[AwsS3Bucket::getObjectDataChunks] object name " << objectName; + LOG(INFO) << "[AwsS3Bucket::getObjectDataChunks] chunk size " << chunkSize; const size_t fileSize = this->getObjectSize(objectName); if (fileSize == 0) { @@ -166,6 +177,7 @@ void AwsS3Bucket::appendToObject( const std::string &objectName, const std::string &data) { + LOG(INFO) << "[AwsS3Bucket::appendToObject] object name " << objectName; const size_t objectSize = this->getObjectSize(objectName); if (objectSize < AWS_MULTIPART_UPLOAD_MINIMUM_CHUNK_SIZE) { std::string currentData = this->getObjectData(objectName); @@ -205,10 +217,12 @@ } void AwsS3Bucket::clearObject(const std::string &objectName) { + LOG(INFO) << "[AwsS3Bucket::clearObject] object name " << objectName; this->writeObject(objectName, ""); } void AwsS3Bucket::removeObject(const std::string &objectName) { + LOG(INFO) << "[AwsS3Bucket::removeObject] object name " << objectName; Aws::S3::Model::DeleteObjectRequest deleteRequest; deleteRequest.SetBucket(this->name); deleteRequest.SetKey(objectName); diff --git a/services/blob/src/BlobServiceImpl.cpp b/services/blob/src/BlobServiceImpl.cpp --- a/services/blob/src/BlobServiceImpl.cpp +++ b/services/blob/src/BlobServiceImpl.cpp @@ -70,6 +70,7 @@ google::protobuf::Empty *response) { grpc::Status status = grpc::Status::OK; const std::string holder = request->holder(); + LOG(INFO) << "[BlobServiceImpl::Remove] holder " << holder; try { std::shared_ptr reverseIndexItem = database::DatabaseManager::getInstance().findReverseIndexItemByHolder( @@ -80,6 +81,7 @@ // TODO handle cleanup here properly // for now the object's being removed right away const std::string blobHash = reverseIndexItem->getBlobHash(); + LOG(INFO) << "[BlobServiceImpl::Remove] blob hash " << blobHash; database::DatabaseManager::getInstance().removeReverseIndexItem(holder); if (database::DatabaseManager::getInstance() .findReverseIndexItemsByHash(reverseIndexItem->getBlobHash()) diff --git a/services/blob/src/DatabaseEntities/BlobItem.cpp b/services/blob/src/DatabaseEntities/BlobItem.cpp --- a/services/blob/src/DatabaseEntities/BlobItem.cpp +++ b/services/blob/src/DatabaseEntities/BlobItem.cpp @@ -2,6 +2,8 @@ #include "Constants.h" +#include + namespace comm { namespace network { namespace database { @@ -25,6 +27,7 @@ } void BlobItem::validate() const { + LOG(INFO) << "[BlobItem::validate]"; if (!this->blobHash.size()) { throw std::runtime_error("blobHash empty"); } @@ -32,8 +35,11 @@ } void BlobItem::assignItemFromDatabase(const AttributeValues &itemFromDB) { + LOG(INFO) << "[BlobItem::assignItemFromDatabase]"; try { this->blobHash = itemFromDB.at(BlobItem::FIELD_BLOB_HASH).GetS(); + LOG(INFO) << "[BlobItem::assignItemFromDatabase] blob hash " + << this->blobHash; this->s3Path = S3Path(itemFromDB.at(BlobItem::FIELD_S3_PATH).GetS()); this->created = std::stoll( std::string(itemFromDB.at(BlobItem::FIELD_CREATED).GetS()).c_str()); diff --git a/services/blob/src/DatabaseEntities/ReverseIndexItem.cpp b/services/blob/src/DatabaseEntities/ReverseIndexItem.cpp --- a/services/blob/src/DatabaseEntities/ReverseIndexItem.cpp +++ b/services/blob/src/DatabaseEntities/ReverseIndexItem.cpp @@ -2,6 +2,8 @@ #include "Constants.h" +#include + namespace comm { namespace network { namespace database { @@ -22,6 +24,7 @@ } void ReverseIndexItem::validate() const { + LOG(INFO) << "[ReverseIndexItem::validate]"; if (!this->holder.size()) { throw std::runtime_error("reverse index empty"); } @@ -32,8 +35,13 @@ void ReverseIndexItem::assignItemFromDatabase( const AttributeValues &itemFromDB) { + LOG(INFO) << "[ReverseIndexItem::assignItemFromDatabase]"; this->holder = itemFromDB.at(ReverseIndexItem::FIELD_HOLDER).GetS(); + LOG(INFO) << "[ReverseIndexItem::assignItemFromDatabase] holder " + << this->holder; this->blobHash = itemFromDB.at(ReverseIndexItem::FIELD_BLOB_HASH).GetS(); + LOG(INFO) << "[ReverseIndexItem::assignItemFromDatabase] blob hash " + << this->blobHash; this->validate(); } diff --git a/services/blob/src/DatabaseManager.cpp b/services/blob/src/DatabaseManager.cpp --- a/services/blob/src/DatabaseManager.cpp +++ b/services/blob/src/DatabaseManager.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include namespace comm { @@ -18,6 +20,8 @@ } void DatabaseManager::putBlobItem(const BlobItem &item) { + LOG(INFO) << "[DatabaseManager::putBlobItem] blob hash " + << item.getBlobHash(); Aws::DynamoDB::Model::PutItemRequest request; request.SetTableName(BlobItem::tableName); request.AddItem( @@ -36,6 +40,7 @@ std::shared_ptr DatabaseManager::findBlobItem(const std::string &blobHash) { + LOG(INFO) << "[DatabaseManager::findBlobItem] blob hash " << blobHash; Aws::DynamoDB::Model::GetItemRequest request; request.AddKey( BlobItem::FIELD_BLOB_HASH, @@ -44,6 +49,7 @@ } void DatabaseManager::removeBlobItem(const std::string &blobHash) { + LOG(INFO) << "[DatabaseManager::removeBlobItem] blob hash " << blobHash; std::shared_ptr item = this->findBlobItem(blobHash); if (item == nullptr) { return; @@ -52,6 +58,8 @@ } void DatabaseManager::putReverseIndexItem(const ReverseIndexItem &item) { + LOG(INFO) << "[DatabaseManager::putReverseIndexItem] holder " + << item.getHolder(); if (this->findReverseIndexItemByHolder(item.getHolder()) != nullptr) { throw std::runtime_error( "An item for the given holder [" + item.getHolder() + @@ -71,6 +79,8 @@ std::shared_ptr DatabaseManager::findReverseIndexItemByHolder(const std::string &holder) { + LOG(INFO) << "[DatabaseManager::findReverseIndexItemByHolder] holder " + << holder; Aws::DynamoDB::Model::GetItemRequest request; request.AddKey( ReverseIndexItem::FIELD_HOLDER, @@ -81,6 +91,8 @@ std::vector> DatabaseManager::findReverseIndexItemsByHash(const std::string &blobHash) { + LOG(INFO) << "[DatabaseManager::findReverseIndexItemsByHash] hash " + << blobHash; std::vector> result; Aws::DynamoDB::Model::QueryRequest req; @@ -107,6 +119,7 @@ } void DatabaseManager::removeReverseIndexItem(const std::string &holder) { + LOG(INFO) << "[DatabaseManager::removeReverseIndexItem] holder " << holder; std::shared_ptr item = findReverseIndexItemByHolder(holder); if (item == nullptr) { diff --git a/services/blob/src/MultiPartUploader.cpp b/services/blob/src/MultiPartUploader.cpp --- a/services/blob/src/MultiPartUploader.cpp +++ b/services/blob/src/MultiPartUploader.cpp @@ -9,6 +9,8 @@ #include +#include + namespace comm { namespace network { @@ -17,6 +19,10 @@ const std::string bucketName, const std::string objectName) : client(client), bucketName(bucketName), objectName(objectName) { + LOG(INFO) << "[MultiPartUploader::MultiPartUploader] bucket name " + << bucketName; + LOG(INFO) << "[MultiPartUploader::MultiPartUploader] object name " + << objectName; this->completeMultipartUploadRequest.SetBucket(this->bucketName); this->completeMultipartUploadRequest.SetKey(this->objectName); @@ -36,6 +42,7 @@ } void MultiPartUploader::addPart(const std::string &part) { + LOG(INFO) << "[MultiPartUploader::addPart] part size " << part.size(); Aws::S3::Model::UploadPartRequest uploadRequest; uploadRequest.SetBucket(this->bucketName); uploadRequest.SetKey(this->objectName); @@ -66,6 +73,7 @@ } void MultiPartUploader::finishUpload() { + LOG(INFO) << "[MultiPartUploader::finishUpload]"; if (!this->completedMultipartUpload.PartsHasBeenSet()) { return; } diff --git a/services/blob/src/Reactors/server/GetReactor.h b/services/blob/src/Reactors/server/GetReactor.h --- a/services/blob/src/Reactors/server/GetReactor.h +++ b/services/blob/src/Reactors/server/GetReactor.h @@ -31,15 +31,19 @@ std::unique_ptr writeResponse(blob::GetResponse *response) override { + LOG(INFO) << "[GetReactor::writeResponse] offset " << this->offset; + LOG(INFO) << "[GetReactor::writeResponse] fileSize " << this->fileSize; if (this->offset >= this->fileSize) { return std::make_unique(grpc::Status::OK); } const size_t nextSize = std::min(this->chunkSize, this->fileSize - this->offset); + LOG(INFO) << "[GetReactor::writeResponse] nextSize " << nextSize; std::string range = "bytes=" + std::to_string(this->offset) + "-" + std::to_string(this->offset + nextSize - 1); + LOG(INFO) << "[GetReactor::writeResponse] range " << range; this->getRequest.SetRange(range); Aws::S3::Model::GetObjectOutcome getOutcome = @@ -56,15 +60,20 @@ buffer << retrievedFile.rdbuf(); std::string result(buffer.str()); response->set_datachunk(result); + LOG(INFO) << "[GetReactor::writeResponse] data chunk size " + << result.size(); this->offset += nextSize; + LOG(INFO) << "[GetReactor::writeResponse] new offset " << this->offset; return nullptr; } void initialize() override { + LOG(INFO) << "[GetReactor::initialize]"; this->s3Path = tools::findS3Path(this->request.holder()); this->fileSize = getBucket(s3Path.getBucketName()).getObjectSize(s3Path.getObjectName()); + LOG(INFO) << "[GetReactor::initialize] file size " << this->fileSize; this->getRequest.SetBucket(this->s3Path.getBucketName()); this->getRequest.SetKey(this->s3Path.getObjectName()); diff --git a/services/blob/src/Reactors/server/PutReactor.h b/services/blob/src/Reactors/server/PutReactor.h --- a/services/blob/src/Reactors/server/PutReactor.h +++ b/services/blob/src/Reactors/server/PutReactor.h @@ -26,17 +26,20 @@ std::unique_ptr handleRequest( blob::PutRequest request, blob::PutResponse *response) override { + LOG(INFO) << "[PutReactor::handleRequest]"; if (this->holder.empty()) { if (request.holder().empty()) { throw std::runtime_error("holder has not been provided"); } this->holder = request.holder(); + LOG(INFO) << "[PutReactor::handleRequest] holder " << this->holder; return nullptr; } if (this->blobHash.empty()) { if (request.blobhash().empty()) { throw std::runtime_error("blob hash has not been provided"); } + LOG(INFO) << "[PutReactor::handleRequest] blob hash " << this->blobHash; this->blobHash = request.blobhash(); this->blobItem = database::DatabaseManager::getInstance().findBlobItem(this->blobHash); @@ -45,6 +48,7 @@ std::make_unique(this->blobItem->getS3Path()); response->set_dataexists(true); this->dataExists = true; + LOG(INFO) << "[PutReactor::handleRequest] data exists"; return std::make_unique( grpc::Status::OK, true); } @@ -53,6 +57,7 @@ this->blobItem = std::make_shared(this->blobHash, *s3Path); response->set_dataexists(false); + LOG(INFO) << "[PutReactor::handleRequest] data does not exist"; return nullptr; } if (request.datachunk().empty()) { @@ -60,6 +65,7 @@ grpc::StatusCode::INVALID_ARGUMENT, "data chunk expected")); } if (this->uploader == nullptr) { + LOG(INFO) << "[PutReactor::handleRequest] initialize MPU"; this->uploader = std::make_unique( getS3Client(), BLOB_BUCKET_NAME, s3Path->getObjectName()); } @@ -72,6 +78,7 @@ } void terminateCallback() override { + LOG(INFO) << "[PutReactor::terminateCallback]"; if (!this->status.status.ok()) { return; } diff --git a/services/blob/src/S3Path.cpp b/services/blob/src/S3Path.cpp --- a/services/blob/src/S3Path.cpp +++ b/services/blob/src/S3Path.cpp @@ -1,5 +1,7 @@ #include "S3Path.h" +#include + #include #include #include @@ -53,6 +55,7 @@ } void S3Path::validate() const { + LOG(INFO) << "[S3Path::MultiPartUploader] validate"; if (!this->bucketName.size()) { throw std::runtime_error("referencing S3 path with an empty object name"); } diff --git a/services/blob/src/S3Tools.cpp b/services/blob/src/S3Tools.cpp --- a/services/blob/src/S3Tools.cpp +++ b/services/blob/src/S3Tools.cpp @@ -6,6 +6,8 @@ #include +#include + #include namespace comm { diff --git a/services/blob/src/Tools.cpp b/services/blob/src/Tools.cpp --- a/services/blob/src/Tools.cpp +++ b/services/blob/src/Tools.cpp @@ -6,6 +6,7 @@ #include "GlobalConstants.h" #include "S3Tools.h" +#include #include #include @@ -23,6 +24,8 @@ } std::string computeHashForFile(const database::S3Path &s3Path) { + LOG(INFO) << "[computeHashForFile] bucket name " << s3Path.getBucketName(); + LOG(INFO) << "[computeHashForFile] object name " << s3Path.getObjectName(); SHA512_CTX ctx; SHA512_Init(&ctx); const std::function callback = @@ -47,6 +50,7 @@ } database::S3Path findS3Path(const std::string &holder) { + LOG(INFO) << "[findS3Path] holder " << holder; std::shared_ptr reverseIndexItem = database::DatabaseManager::getInstance().findReverseIndexItemByHolder( holder); @@ -59,6 +63,7 @@ database::S3Path findS3Path(const database::ReverseIndexItem &reverseIndexItem) { + LOG(INFO) << "[findS3Path] hash " << reverseIndexItem.getBlobHash(); std::shared_ptr blobItem = database::DatabaseManager::getInstance().findBlobItem( reverseIndexItem.getBlobHash());