diff --git a/services/blob/CMakeLists.txt b/services/blob/CMakeLists.txt index e19467390..71be0020c 100644 --- a/services/blob/CMakeLists.txt +++ b/services/blob/CMakeLists.txt @@ -1,111 +1,147 @@ PROJECT(blob C CXX) cmake_minimum_required(VERSION 3.16) +set(CMAKE_CXX_STANDARD 14) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY bin) if(COMMAND cmake_policy) cmake_policy(SET CMP0003 NEW) endif(COMMAND cmake_policy) set(CMAKE_CXX_STANDARD 17) set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) set(WITH_GTEST "Use Google Test" OFF) # FIND LIBS -include(./cmake-components/grpc.cmake) -include(./cmake-components/folly.cmake) -add_subdirectory(./lib/glog) +find_package(glog REQUIRED) +find_package(protobuf REQUIRED) +find_package(gRPC REQUIRED) +find_package(Folly REQUIRED) find_package(AWSSDK REQUIRED COMPONENTS s3 core dynamodb) -find_package(Boost 1.40 COMPONENTS program_options REQUIRED) +find_package(Boost 1.40 + COMPONENTS program_options filesystem context regex system thread + REQUIRE +) find_package(OpenSSL REQUIRED) +find_package(double-conversion REQUIRED) + +if(${CMAKE_CURRENT_SOURCE_DIR} MATCHES "^\/transferred.*") + # Inside the docker build contex + set(_proto_path "grpc") +else() + # Inside repo + set(_proto_path "../../native/cpp/CommonCpp/grpc") +endif() + +# Shared Comm protos +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${_proto_path} + ${CMAKE_CURRENT_BINARY_DIR}/protos + EXCLUDE_FROM_ALL +) + +# Reference native/cpp projects +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../lib/src + ${CMAKE_CURRENT_BINARY_DIR}/common # CMake's build directory + EXCLUDE_FROM_ALL # Don't build everything, just what we need +) # FIND FILES file(GLOB DOUBLE_CONVERSION_SOURCES "./lib/double-conversion/double-conversion/*.cc" ) - -file(GLOB GENERATED_CODE "./_generated/*.cc") - +file(GLOB COMMON_CODE "${CMAKE_CURRENT_SOURCE_DIR}/../lib/src/*.cpp") file(GLOB_RECURSE SOURCE_CODE "./src/*.cpp") -include_directories( - ./src - ./src/server-base-reactors - ./src/DatabaseEntities - ./src/Reactors/ - ./src/Reactors/server - ./src/Reactors/server/base-reactors - ./_generated - ${FOLLY_INCLUDES} - ./lib/double-conversion - ${Boost_INCLUDE_DIR} -) - # SERVER add_executable( blob ${GENERATED_CODE} ${DOUBLE_CONVERSION_SOURCES} ${FOLLY_SOURCES} ${SOURCE_CODE} + ${COMMON_CODE} +) + +target_include_directories( + blob + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/src/server-base-reactors + ${CMAKE_CURRENT_SOURCE_DIR}/src/DatabaseEntities + ${CMAKE_CURRENT_SOURCE_DIR}/src/Reactors/ + ${CMAKE_CURRENT_SOURCE_DIR}/src/Reactors/server + ${CMAKE_CURRENT_SOURCE_DIR}/src/Reactors/server/base-reactors + + # Ideally, we would add comm-services-common as a + # target link library, however, aws-sdk seems to not be able + # to be linked transitively + ${CMAKE_CURRENT_SOURCE_DIR}/../lib/src + + ${Boost_INCLUDE_DIR} ) set( LIBS ${GRPC_LIBS} ${AWSSDK_LINK_LIBRARIES} ${Boost_LIBRARIES} OpenSSL::SSL glog::glog + gRPC::grpc++ + double-conversion::double-conversion + Folly::folly + + comm-blob-grpc + comm-server-base-reactors ) target_link_libraries( blob ${LIBS} ) install( TARGETS blob RUNTIME DESTINATION bin/ ) # TEST if ($ENV{COMM_TEST_SERVICES} MATCHES 1) file(GLOB TEST_CODE "./test/*.cpp") list(FILTER SOURCE_CODE EXCLUDE REGEX "./src/server.cpp") enable_testing() find_package(GTest REQUIRED) include_directories( ${GTEST_INCLUDE_DIR} ./test ) add_executable( runTests ${GENERATED_CODE} ${DOUBLE_CONVERSION_SOURCES} ${FOLLY_SOURCES} ${SOURCE_CODE} ${TEST_CODE} ) target_link_libraries( runTests ${LIBS} gtest gtest_main ) add_test( NAME runTests COMMAND runTests ) endif() diff --git a/services/blob/Dockerfile b/services/blob/Dockerfile index 50322725a..0a93a9a26 100644 --- a/services/blob/Dockerfile +++ b/services/blob/Dockerfile @@ -1,23 +1,27 @@ -FROM commapp/services-base:1.2 +FROM commapp/services-base:1.3 RUN apt-get update && \ apt-get install -y uuid-dev && \ rm -rf /var/lib/apt/lists/* ARG COMM_TEST_SERVICES ARG COMM_SERVICES_SANDBOX ENV COMM_TEST_SERVICES=${COMM_TEST_SERVICES} ENV COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} WORKDIR /transferred -COPY native/cpp/CommonCpp/grpc/protos/blob.proto protos/blob.proto -COPY services/lib/cmake-components cmake-components COPY services/lib/docker/ scripts/ -COPY services/blob/ . -COPY services/lib/src/* src/ +COPY services/blob/ blob +COPY services/lib/src lib/src/ -RUN scripts/build_service.sh +WORKDIR /transferred/blob + +ADD native/cpp/CommonCpp/grpc grpc + +RUN ../scripts/build_service.sh + +WORKDIR /transferred CMD if [ "$COMM_TEST_SERVICES" -eq 1 ]; then scripts/run_tests.sh; else scripts/run_service.sh; fi diff --git a/services/blob/src/BlobServiceImpl.h b/services/blob/src/BlobServiceImpl.h index 3f1bcefde..f665d2f0c 100644 --- a/services/blob/src/BlobServiceImpl.h +++ b/services/blob/src/BlobServiceImpl.h @@ -1,42 +1,42 @@ #pragma once #include "S3Path.h" -#include "../_generated/blob.grpc.pb.h" -#include "../_generated/blob.pb.h" +#include +#include #include #include #include namespace comm { namespace network { class BlobServiceImpl final : public blob::BlobService::CallbackService { void verifyBlobHash( const std::string &expectedBlobHash, const database::S3Path &s3Path); void assignVariableIfEmpty( const std::string &label, std::string &lvalue, const std::string &rvalue); public: BlobServiceImpl(); virtual ~BlobServiceImpl(); grpc::ServerBidiReactor * Put(grpc::CallbackServerContext *context) override; grpc::ServerWriteReactor * Get(grpc::CallbackServerContext *context, const blob::GetRequest *request) override; grpc::ServerUnaryReactor *Remove( grpc::CallbackServerContext *context, const blob::RemoveRequest *request, google::protobuf::Empty *response) override; }; } // namespace network } // namespace comm diff --git a/services/blob/src/DatabaseEntities/DatabaseEntitiesTools.h b/services/blob/src/DatabaseEntities/DatabaseEntitiesTools.h deleted file mode 100644 index 71545a9b2..000000000 --- a/services/blob/src/DatabaseEntities/DatabaseEntitiesTools.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "Item.h" - -#include "BlobItem.h" -#include "ReverseIndexItem.h" - -#include -#include - -namespace comm { -namespace network { -namespace database { - -/** - * Database Structure: - * blob - * blobHash string - * s3Path string - * created timestamp - * reverse_index - * holder string - * blobHash string - */ - -template std::shared_ptr createItemByType() { - static_assert(std::is_base_of::value, "T must inherit from Item"); - return std::make_shared(); -} - -} // namespace database -} // namespace network -} // namespace comm diff --git a/services/blob/src/DatabaseManager.h b/services/blob/src/DatabaseManager.h index 9218eb58a..eb54090c4 100644 --- a/services/blob/src/DatabaseManager.h +++ b/services/blob/src/DatabaseManager.h @@ -1,41 +1,43 @@ #pragma once +#include "BlobItem.h" #include "DatabaseEntitiesTools.h" #include "DatabaseManagerBase.h" #include "DynamoDBTools.h" +#include "ReverseIndexItem.h" #include #include #include #include #include #include #include #include #include namespace comm { namespace network { namespace database { // this class should be thread-safe in case any shared resources appear class DatabaseManager : public DatabaseManagerBase { public: static DatabaseManager &getInstance(); void putBlobItem(const BlobItem &item); std::shared_ptr findBlobItem(const std::string &blobHash); void removeBlobItem(const std::string &blobHash); void putReverseIndexItem(const ReverseIndexItem &item); std::shared_ptr findReverseIndexItemByHolder(const std::string &holder); std::vector> findReverseIndexItemsByHash(const std::string &blobHash); void removeReverseIndexItem(const std::string &holder); }; } // namespace database } // namespace network } // namespace comm diff --git a/services/blob/src/Reactors/server/GetReactor.h b/services/blob/src/Reactors/server/GetReactor.h index b02aa5d02..5b4a2b577 100644 --- a/services/blob/src/Reactors/server/GetReactor.h +++ b/services/blob/src/Reactors/server/GetReactor.h @@ -1,87 +1,87 @@ #pragma once #include "GlobalConstants.h" #include "S3Tools.h" -#include "ServerWriteReactorBase.h" +#include -#include "../_generated/blob.grpc.pb.h" -#include "../_generated/blob.pb.h" +#include +#include #include #include #include namespace comm { namespace network { namespace reactor { class GetReactor : public ServerWriteReactorBase { size_t offset = 0; size_t fileSize = 0; const size_t chunkSize = GRPC_CHUNK_SIZE_LIMIT - GRPC_METADATA_SIZE_PER_MESSAGE; database::S3Path s3Path; Aws::S3::Model::GetObjectRequest getRequest; public: using ServerWriteReactorBase:: ServerWriteReactorBase; std::unique_ptr writeResponse(blob::GetResponse *response) override { if (this->offset >= this->fileSize) { return std::make_unique(grpc::Status::OK); } const size_t nextSize = std::min(this->chunkSize, this->fileSize - this->offset); std::string range = "bytes=" + std::to_string(this->offset) + "-" + std::to_string(this->offset + nextSize - 1); this->getRequest.SetRange(range); Aws::S3::Model::GetObjectOutcome getOutcome = getS3Client()->GetObject(this->getRequest); if (!getOutcome.IsSuccess()) { return std::make_unique( grpc::StatusCode::INTERNAL, getOutcome.GetError().GetMessage()); } Aws::IOStream &retrievedFile = getOutcome.GetResultWithOwnership().GetBody(); std::stringstream buffer; buffer << retrievedFile.rdbuf(); std::string result(buffer.str()); response->set_datachunk(result); this->offset += nextSize; return nullptr; } void initialize() override { this->s3Path = tools::findS3Path(this->request.holder()); AwsS3Bucket bucket = getBucket(this->s3Path.getBucketName()); if (!bucket.isAvailable()) { throw std::runtime_error( "bucket [" + this->s3Path.getBucketName() + "] not available"); } this->fileSize = bucket.getObjectSize(this->s3Path.getObjectName()); if (this->fileSize == 0) { throw std::runtime_error("object empty"); } this->getRequest.SetBucket(this->s3Path.getBucketName()); this->getRequest.SetKey(this->s3Path.getObjectName()); }; void doneCallback() override{}; }; } // namespace reactor } // namespace network } // namespace comm diff --git a/services/blob/src/Tools.h b/services/blob/src/Tools.h index 33aa8cadc..fa7925a2d 100644 --- a/services/blob/src/Tools.h +++ b/services/blob/src/Tools.h @@ -1,28 +1,29 @@ #pragma once #include "DatabaseEntitiesTools.h" +#include "ReverseIndexItem.h" #include "S3Path.h" namespace comm { namespace network { namespace tools { database::S3Path generateS3Path(const std::string &bucketName, const std::string &blobHash); std::string computeHashForFile(const database::S3Path &s3Path); database::S3Path findS3Path(const std::string &holder); database::S3Path findS3Path(const database::ReverseIndexItem &reverseIndexItem); class invalid_argument_error : public std::runtime_error { public: invalid_argument_error(std::string errorMessage) : std::runtime_error(errorMessage) { } }; } // namespace tools } // namespace network } // namespace comm