diff --git a/services/backup/docker-base/Dockerfile b/services/backup/docker-base/Dockerfile index 3f2676171..74a6e7427 100644 --- a/services/backup/docker-base/Dockerfile +++ b/services/backup/docker-base/Dockerfile @@ -1,17 +1,18 @@ FROM commapp/services-base:1.0 ENV SHELL=/bin/bash COPY services/backup/docker-base/contents /transferred RUN apt-get update && apt-get install -y \ libcurl4-openssl-dev \ libssl-dev \ uuid-dev \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* RUN cd / RUN /transferred/install_aws_sdk.sh +RUN /transferred/install_folly.sh CMD /bin/bash diff --git a/services/backup/docker-base/contents/install_aws_sdk.sh b/services/backup/docker-base/contents/install_aws_sdk.sh index 17ee2da8c..c131fcd37 100755 --- a/services/backup/docker-base/contents/install_aws_sdk.sh +++ b/services/backup/docker-base/contents/install_aws_sdk.sh @@ -1,21 +1,20 @@ #!/bin/bash set -e -if [[ -d /usr/lib/aws-sdk-cpp ]] -then - echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0; +if [[ -d /usr/lib/aws-sdk-cpp ]]; then + echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." + exit 0; fi pushd /usr/lib git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp -b 1.9.176 mkdir aws_sdk_build pushd aws_sdk_build cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/local/aws_sdk -DBUILD_ONLY="core;s3;dynamodb" make make install popd # aws_sdk_build popd # /usr/lib diff --git a/services/backup/docker-base/contents/install_folly.sh b/services/backup/docker-base/contents/install_folly.sh new file mode 100755 index 000000000..80d0fd0de --- /dev/null +++ b/services/backup/docker-base/contents/install_folly.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +pushd /usr/lib + +git clone https://github.com/facebook/folly.git --branch v2020.01.13.00 --single-branch +git clone https://github.com/google/glog.git --branch v0.4.0 --single-branch +git clone https://github.com/google/double-conversion.git --branch v3.1.5 --single-branch + +popd # /usr/lib diff --git a/services/backup/docker-server/Dockerfile b/services/backup/docker-server/Dockerfile index 6a9342e94..bbbee9645 100644 --- a/services/backup/docker-server/Dockerfile +++ b/services/backup/docker-server/Dockerfile @@ -1,15 +1,15 @@ -FROM commapp/backup-base:1.0 +FROM commapp/backup-base:1.1 ARG COMM_TEST_SERVICES ARG COMM_SERVICES_DEV_MODE ENV SHELL=/bin/bash ENV COMM_TEST_SERVICES=${COMM_TEST_SERVICES} ENV COMM_SERVICES_DEV_MODE=${COMM_SERVICES_DEV_MODE} COPY services/backup/docker-server/contents /transferred COPY native/cpp/CommonCpp/grpc/protos/backup.proto /transferred/server/protos/backup.proto RUN /transferred/build_server.sh CMD if [ "$COMM_TEST_SERVICES" -eq 1 ]; then /transferred/run_tests.sh; else /transferred/run_server.sh; fi diff --git a/services/backup/docker-server/contents/build_server.sh b/services/backup/docker-server/contents/build_server.sh index 21bd7073c..0e467bc28 100755 --- a/services/backup/docker-server/contents/build_server.sh +++ b/services/backup/docker-server/contents/build_server.sh @@ -1,16 +1,27 @@ #!/bin/bash set -e +# folly hack - https://github.com/facebook/folly/pull/1231 +sed -i 's/#if __has_include()/#if __has_include()/g' /usr/lib/folly/folly/detail/Demangle.h + pushd transferred/server +rm -rf lib +mkdir lib +pushd lib +ln -s /usr/lib/folly +ln -s /usr/lib/glog +ln -s /usr/lib/double-conversion +popd # lib + rm -rf _generated mkdir _generated rm -rf cmake/build mkdir -p cmake/build ./generate.sh ./build.sh popd # transferred/server diff --git a/services/backup/docker-server/contents/server/CMakeLists.txt b/services/backup/docker-server/contents/server/CMakeLists.txt index 6847e2066..556fb8da6 100644 --- a/services/backup/docker-server/contents/server/CMakeLists.txt +++ b/services/backup/docker-server/contents/server/CMakeLists.txt @@ -1,136 +1,125 @@ PROJECT(backup C CXX) cmake_minimum_required(VERSION 3.16) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY bin) if(COMMAND cmake_policy) cmake_policy(SET CMP0003 NEW) endif(COMMAND cmake_policy) set(CMAKE_CXX_STANDARD 17) +set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) +set(WITH_GTEST "Use Google Test" OFF) + # FIND LIBS -set(protobuf_MODULE_COMPATIBLE TRUE) -find_package(Protobuf CONFIG REQUIRED) -message(STATUS "Using protobuf ${Protobuf_VERSION}") - -set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) -set(_PROTOBUF_PROTOC $) - -# gRPC building parameters -set(gRPC_BUILD_CSHARP_EXT OFF) -set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") -# Disable unused plugins -set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) -set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) -set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) -set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) +include(./cmake-components/grpc.cmake) +include(./cmake-components/folly.cmake) +add_subdirectory(./lib/glog) +find_package(AWSSDK REQUIRED COMPONENTS core dynamodb) +find_package(Boost 1.40 COMPONENTS program_options REQUIRED) + +# FIND FILES +file(GLOB DOUBLE_CONVERSION_SOURCES "./lib/double-conversion/double-conversion/*.cc") + if ($ENV{COMM_TEST_SERVICES} MATCHES 1) add_compile_definitions(COMM_TEST_SERVICES) endif() -# Find gRPC installation -# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -find_package(gRPC CONFIG REQUIRED) -message(STATUS "Using gRPC ${gRPC_VERSION}") - -set(_GRPC_GRPCPP gRPC::grpc++) -set(_GRPC_CPP_PLUGIN_EXECUTABLE $) - -set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) - -find_package(AWSSDK REQUIRED COMPONENTS core dynamodb) - -find_package(Boost 1.40 COMPONENTS program_options REQUIRED) - file(GLOB GENERATED_CODE "./_generated/*.cc") set(DEV_SOURCE_CODE "") set(DEV_HEADERS_PATH "") if ($ENV{COMM_SERVICES_DEV_MODE} MATCHES 1) + add_compile_definitions(COMM_SERVICES_DEV_MODE) file(GLOB DEV_SOURCE_CODE "./dev/*.cpp" "./src/*.dev.cpp") set(DEV_HEADERS_PATH "./dev") endif() file(GLOB SOURCE_CODE "./src/*.cpp" "./src/**/*.cpp") list(FILTER SOURCE_CODE EXCLUDE REGEX ".*.dev.cpp$") foreach (ITEM ${DEV_SOURCE_CODE}) string(REPLACE "/" ";" SPLIT_ITEM ${ITEM}) list(GET SPLIT_ITEM -1 FILE_FULL_NAME) string(REPLACE ".dev.cpp" ".cpp" FILE_NAME ${FILE_FULL_NAME}) list(FILTER SOURCE_CODE EXCLUDE REGEX ".*${FILE_NAME}$") list(APPEND SOURCE_CODE "${ITEM}") endforeach() include_directories( ./src ./src/DatabaseEntities ./_generated + ${FOLLY_INCLUDES} + ./lib/double-conversion ${Boost_INCLUDE_DIR} ${DEV_HEADERS_PATH} ) -set( - LIBS - - ${_GRPC_GRPCPP} - ${_PROTOBUF_LIBPROTOBUF} - gRPC::grpc++_reflection - ${AWSSDK_LINK_LIBRARIES} - ${Boost_LIBRARIES} -) - -#SERVER +# SERVER add_executable( backup ${GENERATED_CODE} + ${DOUBLE_CONVERSION_SOURCES} + ${FOLLY_SOURCES} ${SOURCE_CODE} ) +set( + LIBS + + ${GRPC_LIBS} + ${AWSSDK_LINK_LIBRARIES} + ${Boost_LIBRARIES} + glog::glog +) + target_link_libraries( backup ${LIBS} ) install( TARGETS backup RUNTIME DESTINATION bin/ ) # TEST if ($ENV{COMM_TEST_SERVICES} MATCHES 1) file(GLOB TEST_CODE "./test/*.cpp") list(FILTER SOURCE_CODE EXCLUDE REGEX "./src/server.cpp") enable_testing() find_package(GTest REQUIRED) include_directories( ${GTEST_INCLUDE_DIR} ./test ) add_executable( runTests ${GENERATED_CODE} + ${DOUBLE_CONVERSION_SOURCES} + ${FOLLY_SOURCES} ${SOURCE_CODE} ${TEST_CODE} ) target_link_libraries( runTests ${LIBS} gtest gtest_main ) add_test( NAME runTests COMMAND runTests ) endif() diff --git a/services/backup/docker-server/contents/server/build.sh b/services/backup/docker-server/contents/server/build.sh index d1c8002c2..4b180f053 100755 --- a/services/backup/docker-server/contents/server/build.sh +++ b/services/backup/docker-server/contents/server/build.sh @@ -1,21 +1,20 @@ #!/bin/bash set -e NPROC=0 NPROC=$(nproc 2> /dev/null || echo 1) -if [[ $NPROC -eq 1 ]] -then +if [[ $NPROC -eq 1 ]]; then NPROC=$(sysctl -n hw.physicalcpu 2> /dev/null || echo 1) fi echo "building the server (nproc=$NPROC)..." pushd cmake/build cmake ../.. make -j $NPROC popd echo "success - server built" diff --git a/services/backup/docker-server/contents/server/cmake-components/folly.cmake b/services/backup/docker-server/contents/server/cmake-components/folly.cmake new file mode 100644 index 000000000..ad9387051 --- /dev/null +++ b/services/backup/docker-server/contents/server/cmake-components/folly.cmake @@ -0,0 +1,54 @@ +add_definitions( + -DFOLLY_NO_CONFIG=1 + -DFOLLY_HAVE_CLOCK_GETTIME=1 + -DFOLLY_HAVE_MEMRCHR=1 + -DFOLLY_USE_LIBCPP=0 + -DFOLLY_MOBILE=0 +) + +set( + FOLLY_SOURCES + + ./lib/folly/folly/detail/Futex.cpp + ./lib/folly/folly/synchronization/ParkingLot.cpp + ./lib/folly/folly/lang/SafeAssert.cpp + ./lib/folly/folly/FileUtil.cpp + ./lib/folly/folly/Subprocess.cpp + ./lib/folly/folly/File.cpp + ./lib/folly/folly/Format.cpp + ./lib/folly/folly/Conv.cpp + ./lib/folly/folly/io/IOBuf.cpp + ./lib/folly/folly/memory/detail/MallocImpl.cpp + ./lib/folly/folly/ScopeGuard.cpp + ./lib/folly/folly/hash/SpookyHashV2.cpp + ./lib/folly/folly/io/IOBufQueue.cpp + ./lib/folly/folly/lang/Assume.cpp + ./lib/folly/folly/String.cpp + ./lib/folly/folly/portability/SysUio.cpp + ./lib/folly/folly/net/NetOps.cpp + ./lib/folly/folly/synchronization/Hazptr.cpp + ./lib/folly/folly/detail/ThreadLocalDetail.cpp + ./lib/folly/folly/SharedMutex.cpp + ./lib/folly/folly/concurrency/CacheLocality.cpp + ./lib/folly/folly/detail/StaticSingletonManager.cpp + ./lib/folly/folly/executors/ThreadPoolExecutor.cpp + ./lib/folly/folly/executors/GlobalThreadPoolList.cpp + ./lib/folly/folly/Demangle.cpp + ./lib/folly/folly/synchronization/AsymmetricMemoryBarrier.cpp + ./lib/folly/folly/io/async/Request.cpp + ./lib/folly/folly/detail/MemoryIdler.cpp + ./lib/folly/folly/detail/AtFork.cpp + ./lib/folly/folly/Executor.cpp + ./lib/folly/folly/lang/CString.cpp + ./lib/folly/folly/portability/SysMembarrier.cpp + ./lib/folly/folly/container/detail/F14Table.cpp + ./lib/folly/folly/detail/UniqueInstance.cpp + ./lib/folly/folly/executors/QueuedImmediateExecutor.cpp + ./lib/folly/folly/memory/MallctlHelper.cpp +) + +set( + FOLLY_INCLUDES + + ./lib/folly +) diff --git a/services/backup/docker-server/contents/server/cmake-components/grpc.cmake b/services/backup/docker-server/contents/server/cmake-components/grpc.cmake new file mode 100644 index 000000000..2ead89843 --- /dev/null +++ b/services/backup/docker-server/contents/server/cmake-components/grpc.cmake @@ -0,0 +1,31 @@ +# protobuf +set(protobuf_MODULE_COMPATIBLE TRUE) +find_package(Protobuf CONFIG REQUIRED) +message(STATUS "Using protobuf ${Protobuf_VERSION}") + +set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) +set(_PROTOBUF_PROTOC $) + +# gRPC +set(gRPC_BUILD_CSHARP_EXT OFF) +set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") +# Disable unused plugins +set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) +set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) +set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) +set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) + +# Find gRPC installation +find_package(gRPC CONFIG REQUIRED) +message(STATUS "Using gRPC ${gRPC_VERSION}") + +set(_GRPC_GRPCPP gRPC::grpc++) +set(_GRPC_CPP_PLUGIN_EXECUTABLE $) + +set( + GRPC_LIBS + + ${_GRPC_GRPCPP} + ${_PROTOBUF_LIBPROTOBUF} + gRPC::grpc++_reflection +) diff --git a/services/base-image/contents/install_grpc.sh b/services/base-image/contents/install_grpc.sh index d3278d0b8..b0059b904 100755 --- a/services/base-image/contents/install_grpc.sh +++ b/services/base-image/contents/install_grpc.sh @@ -1,35 +1,34 @@ #!/bin/bash set -e echo "installing grpc..." -if [[ -d /usr/lib/grpc ]] -then - echo "grpc already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0; +if [[ -d /usr/lib/grpc ]]; then + echo "grpc already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." + exit 0; fi pushd /usr/lib git clone --recurse-submodules -b v1.39.1 https://github.com/grpc/grpc pushd grpc mkdir -p cmake/build pushd cmake/build cmake -DgRPC_INSTALL=ON \ -DgRPC_BUILD_TESTS=OFF \ ../.. make make install popd # cmake/build pushd third_party/abseil-cpp/ mkdir -p cmake/build pushd cmake/build cmake ../.. make make install popd # cmake/build popd # third_party/abseil-cpp/ popd # grpc popd # /usr/lib diff --git a/services/blob/docker-base/Dockerfile b/services/blob/docker-base/Dockerfile index 277bf599e..264ff9e7c 100644 --- a/services/blob/docker-base/Dockerfile +++ b/services/blob/docker-base/Dockerfile @@ -1,8 +1,18 @@ -FROM commapp/backup-base:1.0 +FROM commapp/services-base:1.0 + +ENV SHELL=/bin/bash COPY services/blob/docker-base/contents /transferred -RUN apt-get update && apt-get install -y libssl-dev && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y \ + libcurl4-openssl-dev \ + libssl-dev \ + uuid-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN cd / +RUN /transferred/install_aws_sdk.sh RUN /transferred/install_folly.sh CMD /bin/bash diff --git a/services/backup/docker-base/contents/install_aws_sdk.sh b/services/blob/docker-base/contents/install_aws_sdk.sh similarity index 63% copy from services/backup/docker-base/contents/install_aws_sdk.sh copy to services/blob/docker-base/contents/install_aws_sdk.sh index 17ee2da8c..c131fcd37 100755 --- a/services/backup/docker-base/contents/install_aws_sdk.sh +++ b/services/blob/docker-base/contents/install_aws_sdk.sh @@ -1,21 +1,20 @@ #!/bin/bash set -e -if [[ -d /usr/lib/aws-sdk-cpp ]] -then - echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0; +if [[ -d /usr/lib/aws-sdk-cpp ]]; then + echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." + exit 0; fi pushd /usr/lib git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp -b 1.9.176 mkdir aws_sdk_build pushd aws_sdk_build cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/local/aws_sdk -DBUILD_ONLY="core;s3;dynamodb" make make install popd # aws_sdk_build popd # /usr/lib diff --git a/services/blob/docker-server/Dockerfile b/services/blob/docker-server/Dockerfile index e44283f9d..14b5e2588 100644 --- a/services/blob/docker-server/Dockerfile +++ b/services/blob/docker-server/Dockerfile @@ -1,16 +1,16 @@ -FROM commapp/blob-base:1.1 +FROM commapp/blob-base:1.2 ARG COMM_SERVICES_DEV_MODE ENV SHELL=/bin/bash ENV COMM_SERVICES_DEV_MODE=${COMM_SERVICES_DEV_MODE} COPY services/blob/docker-server/contents /transferred COPY native/cpp/CommonCpp/grpc/protos/blob.proto /transferred/server/protos/blob.proto ARG COMM_TEST_SERVICES ENV COMM_TEST_SERVICES=${COMM_TEST_SERVICES} RUN /transferred/build_server.sh CMD /transferred/run_server.sh diff --git a/services/blob/docker-server/contents/server/build.sh b/services/blob/docker-server/contents/server/build.sh index d1c8002c2..4b180f053 100755 --- a/services/blob/docker-server/contents/server/build.sh +++ b/services/blob/docker-server/contents/server/build.sh @@ -1,21 +1,20 @@ #!/bin/bash set -e NPROC=0 NPROC=$(nproc 2> /dev/null || echo 1) -if [[ $NPROC -eq 1 ]] -then +if [[ $NPROC -eq 1 ]]; then NPROC=$(sysctl -n hw.physicalcpu 2> /dev/null || echo 1) fi echo "building the server (nproc=$NPROC)..." pushd cmake/build cmake ../.. make -j $NPROC popd echo "success - server built" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 9b54d5c09..354daaa59 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1,54 +1,54 @@ version: "3.9" services: tunnelbroker-server: build: dockerfile: services/tunnelbroker/Dockerfile context: ../ args: - COMM_TEST_SERVICES=${COMM_TEST_SERVICES} image: commapp/tunnelbroker-server:0.2 container_name: tunnelbroker-server ports: - "${COMM_SERVICES_PORT_TUNNELBROKER}:50051" volumes: - $HOME/.aws/config:/root/.aws/config:ro - $HOME/.aws/credentials:/root/.aws/credentials:ro - ./tunnelbroker/tunnelbroker.ini:/root/tunnelbroker/tunnelbroker.ini:ro # backup backup-base: build: dockerfile: services/backup/docker-base/Dockerfile context: ../ - image: commapp/backup-base:1.0 + image: commapp/backup-base:1.1 backup-server: build: dockerfile: services/backup/docker-server/Dockerfile context: ../ args: - COMM_TEST_SERVICES=${COMM_TEST_SERVICES} - COMM_SERVICES_DEV_MODE=${COMM_SERVICES_DEV_MODE} image: commapp/backup-server:0.1 container_name: backup-server ports: - "${COMM_SERVICES_PORT_BACKUP}:50051" volumes: - $HOME/.aws/credentials:/root/.aws/credentials:ro # blob blob-base: build: dockerfile: services/blob/docker-base/Dockerfile context: ../ - image: commapp/blob-base:1.1 + image: commapp/blob-base:1.2 blob-server: build: dockerfile: services/blob/docker-server/Dockerfile context: ../ args: - COMM_TEST_SERVICES=${COMM_TEST_SERVICES} - COMM_SERVICES_DEV_MODE=${COMM_SERVICES_DEV_MODE} image: commapp/blob-server:0.1 container_name: blob-server ports: - "${COMM_SERVICES_PORT_BLOB}:50051" volumes: - $HOME/.aws/credentials:/root/.aws/credentials:ro