diff --git a/services/backup/docker-base/Dockerfile b/services/backup/docker-base/Dockerfile --- a/services/backup/docker-base/Dockerfile +++ b/services/backup/docker-base/Dockerfile @@ -13,5 +13,6 @@ RUN cd / RUN /transferred/install_aws_sdk.sh +RUN /transferred/install_folly.sh CMD /bin/bash diff --git a/services/backup/docker-base/contents/install_aws_sdk.sh b/services/backup/docker-base/contents/install_aws_sdk.sh --- a/services/backup/docker-base/contents/install_aws_sdk.sh +++ b/services/backup/docker-base/contents/install_aws_sdk.sh @@ -2,10 +2,9 @@ set -e -if [[ -d /usr/lib/aws-sdk-cpp ]] -then - echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0; +if [[ -d /usr/lib/aws-sdk-cpp ]]; then + echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." + exit 0; fi pushd /usr/lib diff --git a/services/backup/docker-base/contents/install_folly.sh b/services/backup/docker-base/contents/install_folly.sh new file mode 100755 --- /dev/null +++ b/services/backup/docker-base/contents/install_folly.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +pushd /usr/lib + +git clone https://github.com/facebook/folly.git --branch v2020.01.13.00 --single-branch +git clone https://github.com/google/glog.git --branch v0.4.0 --single-branch +git clone https://github.com/google/double-conversion.git --branch v3.1.5 --single-branch + +popd # /usr/lib diff --git a/services/backup/docker-server/Dockerfile b/services/backup/docker-server/Dockerfile --- a/services/backup/docker-server/Dockerfile +++ b/services/backup/docker-server/Dockerfile @@ -1,4 +1,4 @@ -FROM commapp/backup-base:1.0 +FROM commapp/backup-base:1.1 ARG COMM_TEST_SERVICES ARG COMM_SERVICES_DEV_MODE diff --git a/services/backup/docker-server/contents/build_server.sh b/services/backup/docker-server/contents/build_server.sh --- a/services/backup/docker-server/contents/build_server.sh +++ b/services/backup/docker-server/contents/build_server.sh @@ -2,8 +2,19 @@ set -e +# folly hack - https://github.com/facebook/folly/pull/1231 +sed -i 's/#if __has_include()/#if __has_include()/g' /usr/lib/folly/folly/detail/Demangle.h + pushd transferred/server +rm -rf lib +mkdir lib +pushd lib +ln -s /usr/lib/folly +ln -s /usr/lib/glog +ln -s /usr/lib/double-conversion +popd # lib + rm -rf _generated mkdir _generated diff --git a/services/backup/docker-server/contents/server/CMakeLists.txt b/services/backup/docker-server/contents/server/CMakeLists.txt --- a/services/backup/docker-server/contents/server/CMakeLists.txt +++ b/services/backup/docker-server/contents/server/CMakeLists.txt @@ -10,45 +10,29 @@ set(CMAKE_CXX_STANDARD 17) +set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) +set(WITH_GTEST "Use Google Test" OFF) + # FIND LIBS -set(protobuf_MODULE_COMPATIBLE TRUE) -find_package(Protobuf CONFIG REQUIRED) -message(STATUS "Using protobuf ${Protobuf_VERSION}") - -set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) -set(_PROTOBUF_PROTOC $) - -# gRPC building parameters -set(gRPC_BUILD_CSHARP_EXT OFF) -set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") -# Disable unused plugins -set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) -set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) -set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) -set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) +include(./cmake-components/grpc.cmake) +include(./cmake-components/folly.cmake) +add_subdirectory(./lib/glog) +find_package(AWSSDK REQUIRED COMPONENTS core dynamodb) +find_package(Boost 1.40 COMPONENTS program_options REQUIRED) + +# FIND FILES +file(GLOB DOUBLE_CONVERSION_SOURCES "./lib/double-conversion/double-conversion/*.cc") + if ($ENV{COMM_TEST_SERVICES} MATCHES 1) add_compile_definitions(COMM_TEST_SERVICES) endif() -# Find gRPC installation -# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -find_package(gRPC CONFIG REQUIRED) -message(STATUS "Using gRPC ${gRPC_VERSION}") - -set(_GRPC_GRPCPP gRPC::grpc++) -set(_GRPC_CPP_PLUGIN_EXECUTABLE $) - -set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) - -find_package(AWSSDK REQUIRED COMPONENTS core dynamodb) - -find_package(Boost 1.40 COMPONENTS program_options REQUIRED) - file(GLOB GENERATED_CODE "./_generated/*.cc") set(DEV_SOURCE_CODE "") set(DEV_HEADERS_PATH "") if ($ENV{COMM_SERVICES_DEV_MODE} MATCHES 1) + add_compile_definitions(COMM_SERVICES_DEV_MODE) file(GLOB DEV_SOURCE_CODE "./dev/*.cpp" "./src/*.dev.cpp") set(DEV_HEADERS_PATH "./dev") endif() @@ -68,29 +52,32 @@ ./src ./src/DatabaseEntities ./_generated + ${FOLLY_INCLUDES} + ./lib/double-conversion ${Boost_INCLUDE_DIR} ${DEV_HEADERS_PATH} ) -set( - LIBS - - ${_GRPC_GRPCPP} - ${_PROTOBUF_LIBPROTOBUF} - gRPC::grpc++_reflection - ${AWSSDK_LINK_LIBRARIES} - ${Boost_LIBRARIES} -) - -#SERVER +# SERVER add_executable( backup ${GENERATED_CODE} + ${DOUBLE_CONVERSION_SOURCES} + ${FOLLY_SOURCES} ${SOURCE_CODE} ) +set( + LIBS + + ${GRPC_LIBS} + ${AWSSDK_LINK_LIBRARIES} + ${Boost_LIBRARIES} + glog::glog +) + target_link_libraries( backup @@ -118,6 +105,8 @@ runTests ${GENERATED_CODE} + ${DOUBLE_CONVERSION_SOURCES} + ${FOLLY_SOURCES} ${SOURCE_CODE} ${TEST_CODE} ) diff --git a/services/backup/docker-server/contents/server/build.sh b/services/backup/docker-server/contents/server/build.sh --- a/services/backup/docker-server/contents/server/build.sh +++ b/services/backup/docker-server/contents/server/build.sh @@ -5,8 +5,7 @@ NPROC=0 NPROC=$(nproc 2> /dev/null || echo 1) -if [[ $NPROC -eq 1 ]] -then +if [[ $NPROC -eq 1 ]]; then NPROC=$(sysctl -n hw.physicalcpu 2> /dev/null || echo 1) fi diff --git a/services/backup/docker-server/contents/server/cmake-components/folly.cmake b/services/backup/docker-server/contents/server/cmake-components/folly.cmake new file mode 100644 --- /dev/null +++ b/services/backup/docker-server/contents/server/cmake-components/folly.cmake @@ -0,0 +1,54 @@ +add_definitions( + -DFOLLY_NO_CONFIG=1 + -DFOLLY_HAVE_CLOCK_GETTIME=1 + -DFOLLY_HAVE_MEMRCHR=1 + -DFOLLY_USE_LIBCPP=0 + -DFOLLY_MOBILE=0 +) + +set( + FOLLY_SOURCES + + ./lib/folly/folly/detail/Futex.cpp + ./lib/folly/folly/synchronization/ParkingLot.cpp + ./lib/folly/folly/lang/SafeAssert.cpp + ./lib/folly/folly/FileUtil.cpp + ./lib/folly/folly/Subprocess.cpp + ./lib/folly/folly/File.cpp + ./lib/folly/folly/Format.cpp + ./lib/folly/folly/Conv.cpp + ./lib/folly/folly/io/IOBuf.cpp + ./lib/folly/folly/memory/detail/MallocImpl.cpp + ./lib/folly/folly/ScopeGuard.cpp + ./lib/folly/folly/hash/SpookyHashV2.cpp + ./lib/folly/folly/io/IOBufQueue.cpp + ./lib/folly/folly/lang/Assume.cpp + ./lib/folly/folly/String.cpp + ./lib/folly/folly/portability/SysUio.cpp + ./lib/folly/folly/net/NetOps.cpp + ./lib/folly/folly/synchronization/Hazptr.cpp + ./lib/folly/folly/detail/ThreadLocalDetail.cpp + ./lib/folly/folly/SharedMutex.cpp + ./lib/folly/folly/concurrency/CacheLocality.cpp + ./lib/folly/folly/detail/StaticSingletonManager.cpp + ./lib/folly/folly/executors/ThreadPoolExecutor.cpp + ./lib/folly/folly/executors/GlobalThreadPoolList.cpp + ./lib/folly/folly/Demangle.cpp + ./lib/folly/folly/synchronization/AsymmetricMemoryBarrier.cpp + ./lib/folly/folly/io/async/Request.cpp + ./lib/folly/folly/detail/MemoryIdler.cpp + ./lib/folly/folly/detail/AtFork.cpp + ./lib/folly/folly/Executor.cpp + ./lib/folly/folly/lang/CString.cpp + ./lib/folly/folly/portability/SysMembarrier.cpp + ./lib/folly/folly/container/detail/F14Table.cpp + ./lib/folly/folly/detail/UniqueInstance.cpp + ./lib/folly/folly/executors/QueuedImmediateExecutor.cpp + ./lib/folly/folly/memory/MallctlHelper.cpp +) + +set( + FOLLY_INCLUDES + + ./lib/folly +) diff --git a/services/backup/docker-server/contents/server/cmake-components/grpc.cmake b/services/backup/docker-server/contents/server/cmake-components/grpc.cmake new file mode 100644 --- /dev/null +++ b/services/backup/docker-server/contents/server/cmake-components/grpc.cmake @@ -0,0 +1,31 @@ +# protobuf +set(protobuf_MODULE_COMPATIBLE TRUE) +find_package(Protobuf CONFIG REQUIRED) +message(STATUS "Using protobuf ${Protobuf_VERSION}") + +set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) +set(_PROTOBUF_PROTOC $) + +# gRPC +set(gRPC_BUILD_CSHARP_EXT OFF) +set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") +# Disable unused plugins +set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) +set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) +set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) +set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) + +# Find gRPC installation +find_package(gRPC CONFIG REQUIRED) +message(STATUS "Using gRPC ${gRPC_VERSION}") + +set(_GRPC_GRPCPP gRPC::grpc++) +set(_GRPC_CPP_PLUGIN_EXECUTABLE $) + +set( + GRPC_LIBS + + ${_GRPC_GRPCPP} + ${_PROTOBUF_LIBPROTOBUF} + gRPC::grpc++_reflection +) diff --git a/services/base-image/contents/install_grpc.sh b/services/base-image/contents/install_grpc.sh --- a/services/base-image/contents/install_grpc.sh +++ b/services/base-image/contents/install_grpc.sh @@ -4,10 +4,9 @@ echo "installing grpc..." -if [[ -d /usr/lib/grpc ]] -then - echo "grpc already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0; +if [[ -d /usr/lib/grpc ]]; then + echo "grpc already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." + exit 0; fi pushd /usr/lib diff --git a/services/blob/docker-base/Dockerfile b/services/blob/docker-base/Dockerfile --- a/services/blob/docker-base/Dockerfile +++ b/services/blob/docker-base/Dockerfile @@ -1,8 +1,18 @@ -FROM commapp/backup-base:1.0 +FROM commapp/services-base:1.0 + +ENV SHELL=/bin/bash COPY services/blob/docker-base/contents /transferred -RUN apt-get update && apt-get install -y libssl-dev && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y \ + libcurl4-openssl-dev \ + libssl-dev \ + uuid-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN cd / +RUN /transferred/install_aws_sdk.sh RUN /transferred/install_folly.sh CMD /bin/bash diff --git a/services/backup/docker-base/contents/install_aws_sdk.sh b/services/blob/docker-base/contents/install_aws_sdk.sh copy from services/backup/docker-base/contents/install_aws_sdk.sh copy to services/blob/docker-base/contents/install_aws_sdk.sh --- a/services/backup/docker-base/contents/install_aws_sdk.sh +++ b/services/blob/docker-base/contents/install_aws_sdk.sh @@ -2,10 +2,9 @@ set -e -if [[ -d /usr/lib/aws-sdk-cpp ]] -then - echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0; +if [[ -d /usr/lib/aws-sdk-cpp ]]; then + echo "aws-sdk already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." + exit 0; fi pushd /usr/lib diff --git a/services/blob/docker-server/Dockerfile b/services/blob/docker-server/Dockerfile --- a/services/blob/docker-server/Dockerfile +++ b/services/blob/docker-server/Dockerfile @@ -1,4 +1,4 @@ -FROM commapp/blob-base:1.1 +FROM commapp/blob-base:1.2 ARG COMM_SERVICES_DEV_MODE diff --git a/services/blob/docker-server/contents/server/build.sh b/services/blob/docker-server/contents/server/build.sh --- a/services/blob/docker-server/contents/server/build.sh +++ b/services/blob/docker-server/contents/server/build.sh @@ -5,8 +5,7 @@ NPROC=0 NPROC=$(nproc 2> /dev/null || echo 1) -if [[ $NPROC -eq 1 ]] -then +if [[ $NPROC -eq 1 ]]; then NPROC=$(sysctl -n hw.physicalcpu 2> /dev/null || echo 1) fi diff --git a/services/docker-compose.yml b/services/docker-compose.yml --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -19,7 +19,7 @@ build: dockerfile: services/backup/docker-base/Dockerfile context: ../ - image: commapp/backup-base:1.0 + image: commapp/backup-base:1.1 backup-server: build: dockerfile: services/backup/docker-server/Dockerfile @@ -38,7 +38,7 @@ build: dockerfile: services/blob/docker-base/Dockerfile context: ../ - image: commapp/blob-base:1.1 + image: commapp/blob-base:1.2 blob-server: build: dockerfile: services/blob/docker-server/Dockerfile