diff --git a/services/backup/docker-server/contents/build_server.sh b/services/backup/docker-server/contents/build_server.sh index 0e467bc28..b99faa771 100755 --- a/services/backup/docker-server/contents/build_server.sh +++ b/services/backup/docker-server/contents/build_server.sh @@ -1,27 +1,27 @@ #!/bin/bash set -e # folly hack - https://github.com/facebook/folly/pull/1231 sed -i 's/#if __has_include()/#if __has_include()/g' /usr/lib/folly/folly/detail/Demangle.h -pushd transferred/server +pushd /transferred/server rm -rf lib mkdir lib pushd lib ln -s /usr/lib/folly ln -s /usr/lib/glog ln -s /usr/lib/double-conversion popd # lib rm -rf _generated mkdir _generated rm -rf cmake/build mkdir -p cmake/build ./generate.sh ./build.sh -popd # transferred/server +popd # /transferred/server diff --git a/services/backup/docker-server/contents/server/CMakeLists.txt b/services/backup/docker-server/contents/server/CMakeLists.txt index 556fb8da6..6002419cf 100644 --- a/services/backup/docker-server/contents/server/CMakeLists.txt +++ b/services/backup/docker-server/contents/server/CMakeLists.txt @@ -1,125 +1,124 @@ PROJECT(backup C CXX) cmake_minimum_required(VERSION 3.16) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY bin) if(COMMAND cmake_policy) cmake_policy(SET CMP0003 NEW) endif(COMMAND cmake_policy) set(CMAKE_CXX_STANDARD 17) set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) set(WITH_GTEST "Use Google Test" OFF) # FIND LIBS include(./cmake-components/grpc.cmake) include(./cmake-components/folly.cmake) add_subdirectory(./lib/glog) find_package(AWSSDK REQUIRED COMPONENTS core dynamodb) find_package(Boost 1.40 COMPONENTS program_options REQUIRED) # FIND FILES file(GLOB DOUBLE_CONVERSION_SOURCES "./lib/double-conversion/double-conversion/*.cc") - if ($ENV{COMM_TEST_SERVICES} MATCHES 1) add_compile_definitions(COMM_TEST_SERVICES) endif() file(GLOB GENERATED_CODE "./_generated/*.cc") set(DEV_SOURCE_CODE "") set(DEV_HEADERS_PATH "") if ($ENV{COMM_SERVICES_DEV_MODE} MATCHES 1) add_compile_definitions(COMM_SERVICES_DEV_MODE) file(GLOB DEV_SOURCE_CODE "./dev/*.cpp" "./src/*.dev.cpp") set(DEV_HEADERS_PATH "./dev") endif() file(GLOB SOURCE_CODE "./src/*.cpp" "./src/**/*.cpp") list(FILTER SOURCE_CODE EXCLUDE REGEX ".*.dev.cpp$") foreach (ITEM ${DEV_SOURCE_CODE}) string(REPLACE "/" ";" SPLIT_ITEM ${ITEM}) list(GET SPLIT_ITEM -1 FILE_FULL_NAME) string(REPLACE ".dev.cpp" ".cpp" FILE_NAME ${FILE_FULL_NAME}) list(FILTER SOURCE_CODE EXCLUDE REGEX ".*${FILE_NAME}$") list(APPEND SOURCE_CODE "${ITEM}") endforeach() include_directories( ./src ./src/DatabaseEntities ./_generated ${FOLLY_INCLUDES} ./lib/double-conversion ${Boost_INCLUDE_DIR} ${DEV_HEADERS_PATH} ) # SERVER add_executable( backup ${GENERATED_CODE} ${DOUBLE_CONVERSION_SOURCES} ${FOLLY_SOURCES} ${SOURCE_CODE} ) set( LIBS ${GRPC_LIBS} ${AWSSDK_LINK_LIBRARIES} ${Boost_LIBRARIES} glog::glog ) target_link_libraries( backup ${LIBS} ) install( TARGETS backup RUNTIME DESTINATION bin/ ) # TEST if ($ENV{COMM_TEST_SERVICES} MATCHES 1) file(GLOB TEST_CODE "./test/*.cpp") list(FILTER SOURCE_CODE EXCLUDE REGEX "./src/server.cpp") enable_testing() find_package(GTest REQUIRED) include_directories( ${GTEST_INCLUDE_DIR} ./test ) add_executable( runTests ${GENERATED_CODE} ${DOUBLE_CONVERSION_SOURCES} ${FOLLY_SOURCES} ${SOURCE_CODE} ${TEST_CODE} ) target_link_libraries( runTests ${LIBS} gtest gtest_main ) add_test( NAME runTests COMMAND runTests ) endif() diff --git a/services/backup/docker-server/contents/server/cmake-components/grpc.cmake b/services/backup/docker-server/contents/server/cmake-components/grpc.cmake index 2ead89843..c64424fbd 100644 --- a/services/backup/docker-server/contents/server/cmake-components/grpc.cmake +++ b/services/backup/docker-server/contents/server/cmake-components/grpc.cmake @@ -1,31 +1,22 @@ # protobuf set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) set(_PROTOBUF_PROTOC $) -# gRPC -set(gRPC_BUILD_CSHARP_EXT OFF) -set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") -# Disable unused plugins -set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) -set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) -set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) -set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) - # Find gRPC installation find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) set(_GRPC_CPP_PLUGIN_EXECUTABLE $) set( GRPC_LIBS ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} gRPC::grpc++_reflection ) diff --git a/services/base-image/Dockerfile b/services/base-image/Dockerfile index 09ee3fc7d..9c11f5f57 100644 --- a/services/base-image/Dockerfile +++ b/services/base-image/Dockerfile @@ -1,28 +1,32 @@ FROM ubuntu:20.04 ENV SHELL=/bin/bash -WORKDIR / - -COPY contents /transferred - RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ cmake \ git \ build-essential \ autoconf \ libtool \ pkg-config \ libboost-all-dev \ - libfmt-dev \ + libfmt-dev \ libgtest-dev \ libcurl4-openssl-dev \ libssl-dev \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -RUN /transferred/install_grpc.sh -RUN /transferred/install_aws_sdk.sh -RUN /transferred/install_folly.sh +RUN mkdir -p /transferred/scripts +WORKDIR /transferred/scripts + +COPY contents/install_grpc.sh . +RUN ./install_grpc.sh + +COPY contents/install_aws_sdk.sh . +RUN ./install_aws_sdk.sh + +COPY contents/install_folly.sh . +RUN ./install_folly.sh CMD /bin/bash diff --git a/services/base-image/contents/install_grpc.sh b/services/base-image/contents/install_grpc.sh index b30d1304b..bbdb7a531 100755 --- a/services/base-image/contents/install_grpc.sh +++ b/services/base-image/contents/install_grpc.sh @@ -1,34 +1,47 @@ #!/bin/bash set -e echo "installing grpc..." -if [[ -d /usr/lib/grpc ]]; then - echo "grpc already exists, skipping installation(if the installation seems to be broken, remove this container/image and recreate it)..." - exit 0 -fi +cd /tmp -pushd /usr/lib git clone --recurse-submodules -b v1.39.1 https://github.com/grpc/grpc + pushd grpc mkdir -p cmake/build pushd cmake/build -cmake -DgRPC_INSTALL=ON \ - -DgRPC_BUILD_TESTS=OFF \ - ../.. +cmake \ + -DgRPC_INSTALL=ON \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CPP_PLUGIN=ON \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + ../.. make make install popd # cmake/build +# Explicitly install abseil-cpp because of https://github.com/grpc/grpc/issues/25949 +# This should be removed after upgrading to v1.41 pushd third_party/abseil-cpp/ mkdir -p cmake/build pushd cmake/build -cmake ../.. +cmake \ + -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \ + ../.. make make install popd # cmake/build popd # third_party/abseil-cpp/ popd # grpc -popd # /usr/lib + +rm -rf grpc diff --git a/services/blob/docker-server/contents/build_server.sh b/services/blob/docker-server/contents/build_server.sh index 0e467bc28..b99faa771 100755 --- a/services/blob/docker-server/contents/build_server.sh +++ b/services/blob/docker-server/contents/build_server.sh @@ -1,27 +1,27 @@ #!/bin/bash set -e # folly hack - https://github.com/facebook/folly/pull/1231 sed -i 's/#if __has_include()/#if __has_include()/g' /usr/lib/folly/folly/detail/Demangle.h -pushd transferred/server +pushd /transferred/server rm -rf lib mkdir lib pushd lib ln -s /usr/lib/folly ln -s /usr/lib/glog ln -s /usr/lib/double-conversion popd # lib rm -rf _generated mkdir _generated rm -rf cmake/build mkdir -p cmake/build ./generate.sh ./build.sh -popd # transferred/server +popd # /transferred/server diff --git a/services/blob/docker-server/contents/server/cmake-components/grpc.cmake b/services/blob/docker-server/contents/server/cmake-components/grpc.cmake index 2ead89843..c64424fbd 100644 --- a/services/blob/docker-server/contents/server/cmake-components/grpc.cmake +++ b/services/blob/docker-server/contents/server/cmake-components/grpc.cmake @@ -1,31 +1,22 @@ # protobuf set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) set(_PROTOBUF_PROTOC $) -# gRPC -set(gRPC_BUILD_CSHARP_EXT OFF) -set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") -# Disable unused plugins -set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) -set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) -set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) -set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) - # Find gRPC installation find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) set(_GRPC_CPP_PLUGIN_EXECUTABLE $) set( GRPC_LIBS ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} gRPC::grpc++_reflection ) diff --git a/services/tunnelbroker/CMakeLists.txt b/services/tunnelbroker/CMakeLists.txt index 341a6c4ec..54baa24c3 100644 --- a/services/tunnelbroker/CMakeLists.txt +++ b/services/tunnelbroker/CMakeLists.txt @@ -1,216 +1,207 @@ PROJECT(tunnelbroker C CXX) cmake_minimum_required(VERSION 3.16) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY bin) if(COMMAND cmake_policy) cmake_policy(SET CMP0003 NEW) endif(COMMAND cmake_policy) set(CMAKE_CXX_STANDARD 17) find_package(PkgConfig REQUIRED) # FIND LIBS set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) set(_REFLECTION gRPC::grpc++_reflection) set(_PROTOBUF_PROTOC $) -# gRPC building parameters -set(gRPC_BUILD_CSHARP_EXT OFF) -set(gRPC_SSL_PROVIDER "package" CACHE STRING "SSL library provider") -# Disable unused plugins -set(gRPC_BUILD_GRPC_PHP_PLUGIN OFF) -set(gRPC_BUILD_GRPC_RUBY_PLUGIN OFF) -set(gRPC_BUILD_GRPC_PYTHON_PLUGIN OFF) -set(gRPC_BUILD_GRPC_CSHARP_PLUGIN OFF) - # Find gRPC installation # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. find_package(gRPC CONFIG REQUIRED) message(STATUS "Using gRPC ${gRPC_VERSION}") set(_GRPC_GRPCPP gRPC::grpc++) set(_GRPC_CPP_PLUGIN_EXECUTABLE $) # Find AMQP-CPP installation find_package(amqpcpp CONFIG REQUIRED) # Find Cryptopp installation pkg_check_modules(CRYPTOPP REQUIRED libcryptopp=8.6) # Find Libuv installation pkg_check_modules(LIBUV REQUIRED libuv=1.43.0) set(BUILD_TESTING OFF CACHE BOOL "Turn off tests" FORCE) set(WITH_GFLAGS OFF CACHE BOOL "Turn off gflags" FORCE) find_package(AWSSDK REQUIRED COMPONENTS core dynamodb) find_package(Boost 1.40 COMPONENTS program_options thread system REQUIRED) find_package(OpenSSL REQUIRED) add_subdirectory(./lib/glog) # Generated sources get_filename_component(proto "protos/tunnelbroker.proto" ABSOLUTE) get_filename_component(proto_path "${proto}" PATH) set(GENERATED_BASE_DIR "${CMAKE_CURRENT_BINARY_DIR}/gen") set(GENERATED_CODE_DIR "${GENERATED_BASE_DIR}/_generated") file(MAKE_DIRECTORY ${GENERATED_CODE_DIR}) set(proto_srcs "${GENERATED_CODE_DIR}/tunnelbroker.pb.cc") set(proto_hdrs "${GENERATED_CODE_DIR}/tunnelbroker.pb.h") set(grpc_srcs "${GENERATED_CODE_DIR}/tunnelbroker.grpc.pb.cc") set(grpc_hdrs "${GENERATED_CODE_DIR}/tunnelbroker.grpc.pb.h") add_custom_command( OUTPUT "${proto_srcs}" "${proto_hdrs}" "${grpc_srcs}" "${grpc_hdrs}" COMMAND ${_PROTOBUF_PROTOC} ARGS --grpc_out "${GENERATED_CODE_DIR}" --cpp_out "${GENERATED_CODE_DIR}" -I "${proto_path}" --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" "${proto}" DEPENDS "${proto}" ) file(GLOB SOURCE_CODE "./src/*.cpp" "./src/**/*.cpp") set(GENERATED_CODE ${proto_srcs} ${proto_hdrs} ${grpc_srcs} ${grpc_hdrs}) file(GLOB DOUBLE_CONVERSION_SOURCES "./lib/double-conversion/double-conversion/*.cc") add_definitions( -DFOLLY_NO_CONFIG=1 -DFOLLY_HAVE_CLOCK_GETTIME=1 -DFOLLY_HAVE_MEMRCHR=1 -DFOLLY_USE_LIBCPP=0 -DFOLLY_MOBILE=0 ) include_directories( ./src ./src/Database ./src/DeliveryBroker ./src/Service ./src/Tools ./src/Amqp ${GENERATED_BASE_DIR} ./lib/folly ./lib/double-conversion ${Boost_INCLUDE_DIR} ${CRYPTOPP_INCLUDE_DIRS} ${LIBUV_INCLUDE_DIRS} ) set( SOURCE_CODE # folly ./lib/folly/folly/detail/Futex.cpp ./lib/folly/folly/synchronization/ParkingLot.cpp ./lib/folly/folly/lang/SafeAssert.cpp ./lib/folly/folly/FileUtil.cpp ./lib/folly/folly/Subprocess.cpp ./lib/folly/folly/File.cpp ./lib/folly/folly/Format.cpp ./lib/folly/folly/Conv.cpp ./lib/folly/folly/io/IOBuf.cpp ./lib/folly/folly/memory/detail/MallocImpl.cpp ./lib/folly/folly/ScopeGuard.cpp ./lib/folly/folly/hash/SpookyHashV2.cpp ./lib/folly/folly/io/IOBufQueue.cpp ./lib/folly/folly/lang/Assume.cpp ./lib/folly/folly/String.cpp ./lib/folly/folly/portability/SysUio.cpp ./lib/folly/folly/net/NetOps.cpp ./lib/folly/folly/synchronization/Hazptr.cpp ./lib/folly/folly/detail/ThreadLocalDetail.cpp ./lib/folly/folly/SharedMutex.cpp ./lib/folly/folly/concurrency/CacheLocality.cpp ./lib/folly/folly/detail/StaticSingletonManager.cpp ./lib/folly/folly/executors/ThreadPoolExecutor.cpp ./lib/folly/folly/executors/GlobalThreadPoolList.cpp ./lib/folly/folly/Demangle.cpp ./lib/folly/folly/synchronization/AsymmetricMemoryBarrier.cpp ./lib/folly/folly/io/async/Request.cpp ./lib/folly/folly/detail/MemoryIdler.cpp ./lib/folly/folly/detail/AtFork.cpp ./lib/folly/folly/Executor.cpp ./lib/folly/folly/lang/CString.cpp ./lib/folly/folly/portability/SysMembarrier.cpp ./lib/folly/folly/container/detail/F14Table.cpp ./lib/folly/folly/detail/UniqueInstance.cpp ./lib/folly/folly/executors/QueuedImmediateExecutor.cpp ./lib/folly/folly/memory/MallctlHelper.cpp ${DOUBLE_CONVERSION_SOURCES} ${GENERATED_CODE} ${SOURCE_CODE} ) set( LIBS ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} ${_REFLECTION} ${AWSSDK_LINK_LIBRARIES} ${CRYPTOPP_LIBRARIES} ${LIBUV_LIBRARIES} ${Boost_LIBRARIES} amqpcpp OpenSSL::SSL glog::glog ) #SERVER add_executable( tunnelbroker ${GENERATED_CODE} ${SOURCE_CODE} ) target_link_libraries( tunnelbroker ${LIBS} ) install( TARGETS tunnelbroker RUNTIME DESTINATION bin/ ) # TEST if ($ENV{COMM_TEST_SERVICES} MATCHES 1) file(GLOB TEST_CODE "./test/*.cpp") list(FILTER SOURCE_CODE EXCLUDE REGEX "./src/server.cpp") enable_testing() find_package(GTest REQUIRED) include_directories(${GTEST_INCLUDE_DIR}) add_executable( runTests ${SOURCE_CODE} ${TEST_CODE} ) target_link_libraries( runTests ${LIBS} gtest gtest_main ) add_test( NAME runTests COMMAND runTests ) endif() diff --git a/services/tunnelbroker/Dockerfile b/services/tunnelbroker/Dockerfile index ed4102997..8bf194fca 100644 --- a/services/tunnelbroker/Dockerfile +++ b/services/tunnelbroker/Dockerfile @@ -1,30 +1,29 @@ FROM commapp/services-base:1.1 ARG MAKE_JOBS=4 ENV MAKEFLAGS="-j${MAKE_JOBS}" -RUN mkdir /transferred/scripts WORKDIR /transferred/scripts # Install SDKs COPY services/tunnelbroker/docker/install_amqp_cpp.sh . RUN ./install_amqp_cpp.sh COPY services/tunnelbroker/docker/install_cryptopp.sh . RUN ./install_cryptopp.sh COPY services/tunnelbroker/docker/install_libuv.sh . RUN ./install_libuv.sh ARG COMM_TEST_SERVICES ENV COMM_TEST_SERVICES=${COMM_TEST_SERVICES} WORKDIR /transferred COPY native/cpp/CommonCpp/grpc/protos/tunnelbroker.proto protos/tunnelbroker.proto COPY services/tunnelbroker/docker/ scripts/ COPY services/tunnelbroker/ . RUN scripts/build_server.sh CMD if [ "$COMM_TEST_SERVICES" -eq 1 ]; then scripts/run_tests.sh; else scripts/run_server.sh; fi